From pypy.commits at gmail.com Fri Mar 1 05:10:56 2019 From: pypy.commits at gmail.com (mattip) Date: Fri, 01 Mar 2019 02:10:56 -0800 (PST) Subject: [pypy-commit] pypy default: fix test from 275fd99e1c23 - __args__.keywords can be None or an empty list Message-ID: <5c790530.1c69fb81.63602.fdfd@mx.google.com> Author: Matti Picus Branch: Changeset: r96195:739b9c69e036 Date: 2019-02-28 23:25 +0200 http://bitbucket.org/pypy/pypy/changeset/739b9c69e036/ Log: fix test from 275fd99e1c23 - __args__.keywords can be None or an empty list diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -46,15 +46,15 @@ _dealloc(space, py_obj) def w_kwargs_from_args(space, __args__): - w_kwargs = None - if __args__.keywords: - # CCC: we should probably have a @jit.look_inside_iff if the - # keyword count is constant, as we do in Arguments.unpack - w_kwargs = space.newdict() - for i in range(len(__args__.keywords)): - key = __args__.keywords[i] - w_obj = __args__.keywords_w[i] - space.setitem(w_kwargs, space.newtext(key), w_obj) + if __args__.keywords is None: + return None + # CCC: we should probably have a @jit.look_inside_iff if the + # keyword count is constant, as we do in Arguments.unpack + w_kwargs = space.newdict() + for i in range(len(__args__.keywords)): + key = __args__.keywords[i] + w_obj = __args__.keywords_w[i] + space.setitem(w_kwargs, space.newtext(key), w_obj) return w_kwargs class W_PyCFunctionObject(W_Root): From pypy.commits at gmail.com Fri Mar 1 05:11:00 2019 From: pypy.commits at gmail.com (mattip) Date: Fri, 01 Mar 2019 02:11:00 -0800 (PST) Subject: [pypy-commit] pypy newmemoryview-app-level: merge default into branch Message-ID: <5c790534.1c69fb81.9ba28.57dd@mx.google.com> Author: Matti Picus Branch: newmemoryview-app-level Changeset: r96196:22d553bf29f9 Date: 2019-03-01 07:32 +0200 http://bitbucket.org/pypy/pypy/changeset/22d553bf29f9/ Log: merge default into branch diff too long, truncating to 2000 out of 203400 lines diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py --- a/pypy/interpreter/astcompiler/test/test_compiler.py +++ b/pypy/interpreter/astcompiler/test/test_compiler.py @@ -1204,3 +1204,17 @@ source = 'def f(): %s' % source counts = self.count_instructions(source) assert ops.BINARY_POWER not in counts + + def test_constant_tuples(self): + source = """def f(): + return ((u"a", 1), 2) + """ + counts = self.count_instructions(source) + assert ops.BUILD_TUPLE not in counts + # also for bytes + source = """def f(): + return ((b"a", 5), 5, 7, 8) + """ + counts = self.count_instructions(source) + assert ops.BUILD_TUPLE not in counts + diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -46,15 +46,15 @@ _dealloc(space, py_obj) def w_kwargs_from_args(space, __args__): - w_kwargs = None - if __args__.keywords: - # CCC: we should probably have a @jit.look_inside_iff if the - # keyword count is constant, as we do in Arguments.unpack - w_kwargs = space.newdict() - for i in range(len(__args__.keywords)): - key = __args__.keywords[i] - w_obj = __args__.keywords_w[i] - space.setitem(w_kwargs, space.newtext(key), w_obj) + if __args__.keywords is None: + return None + # CCC: we should probably have a @jit.look_inside_iff if the + # keyword count is constant, as we do in Arguments.unpack + w_kwargs = space.newdict() + for i in range(len(__args__.keywords)): + key = __args__.keywords[i] + w_obj = __args__.keywords_w[i] + space.setitem(w_kwargs, space.newtext(key), w_obj) return w_kwargs class W_PyCFunctionObject(W_Root): diff --git a/pypy/module/cpyext/test/test_methodobject.py b/pypy/module/cpyext/test/test_methodobject.py --- a/pypy/module/cpyext/test/test_methodobject.py +++ b/pypy/module/cpyext/test/test_methodobject.py @@ -108,6 +108,7 @@ assert mod.getarg_KW(a=3, b=4) == ((), {'a': 3, 'b': 4}) assert mod.getarg_KW(1, 2, a=3, b=4) == ((1, 2), {'a': 3, 'b': 4}) assert mod.getarg_KW.__name__ == "getarg_KW" + assert mod.getarg_KW(*(), **{}) == ((), {}) def test_func_attributes(self): mod = self.import_extension('MyModule', [ diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -1190,6 +1190,11 @@ os = self.posix with open(self.path, "w") as f: f.write("this is a rename test") + str_name = str(self.pdir) + '/test_rename.txt' + os.rename(self.path, str_name) + with open(str_name) as f: + assert f.read() == 'this is a rename test' + os.rename(str_name, self.path) unicode_name = str(self.udir) + u'/test\u03be.txt' os.rename(self.path, unicode_name) with open(unicode_name) as f: diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -1273,8 +1273,6 @@ win32traits = make_win32_traits(traits) path1 = traits.as_str0(path1) path2 = traits.as_str0(path2) - assert isinstance(path1, unicode) - assert isinstance(path2, unicode) if not win32traits.MoveFileEx(path1, path2, 0): raise rwin32.lastSavedWindowsError() @@ -1285,8 +1283,6 @@ win32traits = make_win32_traits(traits) path1 = traits.as_str0(path1) path2 = traits.as_str0(path2) - assert isinstance(path1, unicode) - assert isinstance(path2, unicode) ret = win32traits.MoveFileEx(path1, path2, win32traits.MOVEFILE_REPLACE_EXISTING) if not ret: diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py --- a/rpython/rlib/rutf8.py +++ b/rpython/rlib/rutf8.py @@ -153,6 +153,7 @@ """Gives the position of the previous codepoint. 'pos' must not be zero. """ + assert pos != 0 pos -= 1 if pos >= len(code): # for the case where pos - 1 == len(code): assert pos >= 0 @@ -811,6 +812,17 @@ (0xF0 << 18) + (0x80 << 12) + (0x80 << 6) + 0x80 ) assert False, "unreachable" +class Utf8StringPosIterator(object): + def __init__(self, utf8s): + self.it = Utf8StringIterator(utf8s) + + def __iter__(self): + return self + + def next(self): + pos = self.it.get_pos() + return (self.it.next(), pos) + def decode_latin_1(s): if len(s) == 0: diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py --- a/rpython/rlib/test/test_rsocket.py +++ b/rpython/rlib/test/test_rsocket.py @@ -416,7 +416,7 @@ assert isinstance(lst, list) found = False for family, socktype, protocol, canonname, addr in lst: - if addr.get_host() in ('104.130.43.121', '23.253.135.79'): + if addr.get_host() in ('104.130.43.121', '23.253.135.79', '45.55.99.72'): found = True elif family == AF_INET: print 'pydotorg changed to', addr.get_host() diff --git a/rpython/rlib/test/test_rutf8.py b/rpython/rlib/test/test_rutf8.py --- a/rpython/rlib/test/test_rutf8.py +++ b/rpython/rlib/test/test_rutf8.py @@ -212,3 +212,16 @@ for c in u: l.append(unichr(c)) assert list(arg) == l + + at given(strategies.text()) +def test_utf8_iterator_pos(arg): + utf8s = arg.encode('utf8') + u = rutf8.Utf8StringPosIterator(utf8s) + l = [] + i = 0 + for c, pos in u: + l.append(unichr(c)) + assert c == rutf8.codepoint_at_pos(utf8s, pos) + assert pos == i + i = rutf8.next_codepoint_pos(utf8s, i) + assert list(arg) == l diff --git a/rpython/rlib/unicodedata/generate_unicodedb.py b/rpython/rlib/unicodedata/generate_unicodedb.py --- a/rpython/rlib/unicodedata/generate_unicodedb.py +++ b/rpython/rlib/unicodedata/generate_unicodedb.py @@ -505,43 +505,36 @@ if base_mod is None: triegenerator.build_compression_tree(outfile, names) print >> outfile, "# the following dictionary is used by modules that take this as a base" + print >> outfile, "# only used by generate_unicodedb, not after translation" print >> outfile, "_orig_names = {" for name, code in sorted_names_codes: print >> outfile, "%r: %r," % (name, code) print >> outfile, "}" else: - print >> outfile, '_names = {' + corrected_names = [] + for name, code in sorted_names_codes: try: if base_mod.lookup_charcode(code) == name: continue except KeyError: pass - print >> outfile, '%r: %r,' % (code, name) - print >> outfile, '}' + corrected_names.append((name, code)) + corrected_names_dict = dict(corrected_names) + triegenerator.build_compression_tree(outfile, corrected_names_dict) - - print >> outfile, '_names_corrected = {' + removed_names = [] for name, code in sorted(base_mod._orig_names.iteritems()): if name not in names: - print >> outfile, '%r: None,' % code - print >> outfile, '}' - - print >> outfile, '_code_by_name = {' - corrected = {} - for name, code in sorted_names_codes: - try: - if base_mod.lookup_charcode(code) == name: - continue - except KeyError: - pass - print >> outfile, '%r: %r,' % (name, code) + removed_names.append((name, code)) + print >> outfile, '_names_corrected = {' + for name, code in removed_names: + print >> outfile, '%r: None,' % code print >> outfile, '}' print >> outfile, '_code_by_name_corrected = {' - for name, code in sorted(base_mod._orig_names.iteritems()): - if name not in names: - print >> outfile, '%r: None,' % name + for name, code in removed_names: + print >> outfile, '%r: None,' % name print >> outfile, '}' @@ -657,7 +650,7 @@ code = trie_lookup(name) else: try: - code = _code_by_name[name] + code = trie_lookup(name) except KeyError: if name not in _code_by_name_corrected: code = base_mod.trie_lookup(name) @@ -686,7 +679,7 @@ return lookup_charcode(code) else: try: - return _names[code] + return lookup_charcode(code) except KeyError: if code not in _names_corrected: return base_mod.lookup_charcode(code) @@ -895,7 +888,7 @@ else: return None ''' % dict(start=table.NAMED_SEQUENCES_START) - + # aliases print >> outfile, '_name_aliases = [' for name, char in table.aliases: diff --git a/rpython/rlib/unicodedata/triegenerator.py b/rpython/rlib/unicodedata/triegenerator.py --- a/rpython/rlib/unicodedata/triegenerator.py +++ b/rpython/rlib/unicodedata/triegenerator.py @@ -76,7 +76,9 @@ charnode = 0 while 0 <= charnode < 0xffff: # 16bit number, 0xffff = None charnode *= 3 - leftright, parentstr, codepoint = _charnodes[charnode:charnode+3] + leftright = _charnodes[charnode] + parentstr = _charnodes[charnode + 1] + codepoint = _charnodes[charnode + 2] if leftright < 0: # XXX assumes msb is sign @@ -91,7 +93,7 @@ else: parent = (parentstr & 0x7fffffff) >> %(STRIDXBITS)d stridx = parentstr & ((1 << %(STRIDXBITS)d) - 1) - + strlen = ord(_stringtable[stridx]) substring = _stringtable[stridx+1:stridx+1+strlen] @@ -109,7 +111,9 @@ prevnode = -1 while 0 <= charnode < 0xffff: # 16bit number, 0xffff = None charnode *= 3 - leftright, parentstr, codepoint = _charnodes[charnode:charnode+3] + leftright = _charnodes[charnode] + parentstr = _charnodes[charnode + 1] + codepoint = _charnodes[charnode + 2] if leftright < 0: # XXX assumes msg is sign @@ -128,13 +132,14 @@ stridx = parentstr & ((1<<%(STRIDXBITS)d)-1) strlen = ord(_stringtable[stridx]) substring = _stringtable[stridx+1:stridx+1+strlen] - res.insert(0, substring) + res.append(substring) prevnode = charnode // 3 charnode = parent + res.reverse() return ''.join(res) - + """ % globals() def findranges(d): @@ -165,6 +170,11 @@ return collapsed def build_compression_tree(outfile, ucdata): + print >> outfile, "#" + "_" * 60 + print >> outfile, "# output from build_compression_tree" + if not ucdata: + print >> outfile, empty_trie_functions + return print >> outfile, classdef reversedict = {} @@ -180,7 +190,7 @@ print >> outfile, "%r" % (chr(strlen) + string) stringidx[string] = stridx stridx += strlen + 1 - + print >> outfile, ")" assert stridx < (1< %d chars" % ( @@ -190,7 +200,7 @@ nodelist = [] maxidx = 0 nodes = [rootnode] - + while nodes: n = nodes.pop() nodelist.append(n) @@ -198,28 +208,31 @@ nodes.append(n.left) if n.right: nodes.append(n.right) - + nodelist.sort(key=lambda x: x.index) newnodes = [] map(newnodes.extend, (n.as_list(stringidx) for n in nodelist)) print >> outfile, "_charnodes =", pprint.pprint(newnodes, stream=outfile) - + function = ["def lookup_charcode(code):", " res = -1"] ranges = collapse_ranges(findranges(reversedict)) + prefix = "" for low, high in ranges: if high - low <= MINLIST: for code in range(low, high + 1): if code in reversedict: function.append( - " if code == %d: res = %s" % - (code, reversedict[code].index)) + " %sif code == %d: res = %s" % + (prefix, code, reversedict[code].index)) + prefix = "el" continue function.append( - " if %d <= code <= %d: res = _charnames_%d[code-%d]" % ( - low, high, low, low)) + " %sif %d <= code <= %d: res = _charnames_%d[code-%d]" % ( + prefix, low, high, low, low)) + prefix = "el" print >> outfile, "_charnames_%d = [" % (low,) for code in range(low, high + 1): @@ -234,6 +247,8 @@ "", ]) print >> outfile, '\n'.join(function) + print >> outfile, "# end output from build_compression_tree" + print >> outfile, "#" + "_" * 60 return rootnode def gen_compression_tree(stringlist, ucdata, reversedict, parent=None, parent_str="", left=False): @@ -244,8 +259,8 @@ for string in stringlist: for stop in range(1, len(string) + 1): codes[string[:stop]] = codes.get(string[:stop], 0) + 1 - - s = [((freq), code) for (code, freq) in codes.iteritems()] + + s = [((freq), code) for (code, freq) in codes.iteritems()] s.sort() if not s: return None @@ -306,3 +321,10 @@ import sys build_compression_tree(sys.stdout, testdata) + +empty_trie_functions = """ +def trie_lookup(name): + raise KeyError +def lookup_charcode(code): + raise KeyError +""" diff --git a/rpython/rlib/unicodedata/unicodedb_3_2_0.py b/rpython/rlib/unicodedata/unicodedb_3_2_0.py --- a/rpython/rlib/unicodedata/unicodedb_3_2_0.py +++ b/rpython/rlib/unicodedata/unicodedb_3_2_0.py @@ -8,8 +8,14 @@ import unicodedb_5_2_0 as base_mod version = '3.2.0' -_names = { -} +#____________________________________________________________ +# output from build_compression_tree + +def trie_lookup(name): + raise KeyError +def lookup_charcode(code): + raise KeyError + _names_corrected = { 9190: None, 65794: None, @@ -8370,8 +8376,6 @@ 11825: None, 983048: None, } -_code_by_name = { -} _code_by_name_corrected = { 'AC CURRENT': None, 'AEGEAN CHECK MARK': None, @@ -16798,7 +16802,7 @@ code = trie_lookup(name) else: try: - code = _code_by_name[name] + code = trie_lookup(name) except KeyError: if name not in _code_by_name_corrected: code = base_mod.trie_lookup(name) @@ -16827,7 +16831,7 @@ return lookup_charcode(code) else: try: - return _names[code] + return lookup_charcode(code) except KeyError: if code not in _names_corrected: return base_mod.lookup_charcode(code) diff --git a/rpython/rlib/unicodedata/unicodedb_5_2_0.py b/rpython/rlib/unicodedata/unicodedb_5_2_0.py --- a/rpython/rlib/unicodedata/unicodedb_5_2_0.py +++ b/rpython/rlib/unicodedata/unicodedb_5_2_0.py @@ -8,12 +8,16 @@ base_mod = None version = '5.2.0' +#____________________________________________________________ +# output from build_compression_tree def trie_lookup(name): charnode = 0 while 0 <= charnode < 0xffff: # 16bit number, 0xffff = None charnode *= 3 - leftright, parentstr, codepoint = _charnodes[charnode:charnode+3] + leftright = _charnodes[charnode] + parentstr = _charnodes[charnode + 1] + codepoint = _charnodes[charnode + 2] if leftright < 0: # XXX assumes msb is sign @@ -28,7 +32,7 @@ else: parent = (parentstr & 0x7fffffff) >> 16 stridx = parentstr & ((1 << 16) - 1) - + strlen = ord(_stringtable[stridx]) substring = _stringtable[stridx+1:stridx+1+strlen] @@ -46,7 +50,9 @@ prevnode = -1 while 0 <= charnode < 0xffff: # 16bit number, 0xffff = None charnode *= 3 - leftright, parentstr, codepoint = _charnodes[charnode:charnode+3] + leftright = _charnodes[charnode] + parentstr = _charnodes[charnode + 1] + codepoint = _charnodes[charnode + 2] if leftright < 0: # XXX assumes msg is sign @@ -65,13 +71,14 @@ stridx = parentstr & ((1<<16)-1) strlen = ord(_stringtable[stridx]) substring = _stringtable[stridx+1:stridx+1+strlen] - res.insert(0, substring) + res.append(substring) prevnode = charnode // 3 charnode = parent + res.reverse() return ''.join(res) - + _stringtable = ( '\x01 ' @@ -114455,149 +114462,152 @@ def lookup_charcode(code): res = -1 if 32 <= code <= 126: res = _charnames_32[code-32] - if 160 <= code <= 1317: res = _charnames_160[code-160] - if 1329 <= code <= 1479: res = _charnames_1329[code-1329] - if 1488 <= code <= 1524: res = _charnames_1488[code-1488] - if 1536 <= code <= 1969: res = _charnames_1536[code-1536] - if 1984 <= code <= 2110: res = _charnames_1984[code-1984] - if 2304 <= code <= 2510: res = _charnames_2304[code-2304] - if 2519 <= code <= 2641: res = _charnames_2519[code-2519] - if code == 2649: res = 22369 - if code == 2650: res = 22374 - if code == 2651: res = 22402 - if code == 2652: res = 22383 - if code == 2654: res = 22408 - if 2662 <= code <= 2677: res = _charnames_2662[code-2662] - if 2689 <= code <= 2768: res = _charnames_2689[code-2689] - if 2784 <= code <= 2801: res = _charnames_2784[code-2784] - if 2817 <= code <= 2893: res = _charnames_2817[code-2817] - if 2902 <= code <= 2929: res = _charnames_2902[code-2902] - if 2946 <= code <= 3031: res = _charnames_2946[code-2946] - if 3046 <= code <= 3149: res = _charnames_3046[code-3046] - if 3157 <= code <= 3183: res = _charnames_3157[code-3157] - if 3192 <= code <= 3277: res = _charnames_3192[code-3192] - if code == 3285: res = 23172 - if code == 3286: res = 23215 - if 3294 <= code <= 3314: res = _charnames_3294[code-3294] - if 3330 <= code <= 3405: res = _charnames_3330[code-3330] - if code == 3415: res = 6443 - if 3424 <= code <= 3551: res = _charnames_3424[code-3424] - if code == 3570: res = 19930 - if code == 3571: res = 19929 - if code == 3572: res = 19944 - if 3585 <= code <= 3675: res = _charnames_3585[code-3585] - if 3713 <= code <= 3805: res = _charnames_3713[code-3713] - if 3840 <= code <= 4056: res = _charnames_3840[code-3840] - if 4096 <= code <= 4293: res = _charnames_4096[code-4096] - if 4304 <= code <= 5108: res = _charnames_4304[code-4304] - if 5120 <= code <= 5872: res = _charnames_5120[code-5120] - if 5888 <= code <= 5908: res = _charnames_5888[code-5888] - if 5920 <= code <= 5942: res = _charnames_5920[code-5920] - if 5952 <= code <= 5971: res = _charnames_5952[code-5952] - if 5984 <= code <= 6003: res = _charnames_5984[code-5984] - if 6016 <= code <= 6263: res = _charnames_6016[code-6016] - if 6272 <= code <= 6389: res = _charnames_6272[code-6272] - if 6400 <= code <= 6516: res = _charnames_6400[code-6400] - if 6528 <= code <= 6829: res = _charnames_6528[code-6528] - if 6912 <= code <= 7097: res = _charnames_6912[code-6912] - if 7168 <= code <= 7295: res = _charnames_7168[code-7168] - if 7376 <= code <= 7410: res = _charnames_7376[code-7376] - if 7424 <= code <= 7654: res = _charnames_7424[code-7424] - if 7677 <= code <= 8340: res = _charnames_7677[code-7677] - if 8352 <= code <= 8376: res = _charnames_8352[code-8352] - if 8400 <= code <= 8432: res = _charnames_8400[code-8400] - if 8448 <= code <= 9192: res = _charnames_8448[code-8448] - if 9216 <= code <= 9254: res = _charnames_9216[code-9216] - if 9280 <= code <= 9290: res = _charnames_9280[code-9280] - if 9312 <= code <= 11097: res = _charnames_9312[code-9312] - if 11264 <= code <= 11505: res = _charnames_11264[code-11264] - if 11513 <= code <= 11557: res = _charnames_11513[code-11513] - if 11568 <= code <= 11621: res = _charnames_11568[code-11568] - if code == 11631: res = 13355 - if 11648 <= code <= 11670: res = _charnames_11648[code-11648] - if 11680 <= code <= 11825: res = _charnames_11680[code-11680] - if 11904 <= code <= 12019: res = _charnames_11904[code-11904] - if 12032 <= code <= 12245: res = _charnames_12032[code-12032] - if 12272 <= code <= 12727: res = _charnames_12272[code-12272] - if 12736 <= code <= 12771: res = _charnames_12736[code-12736] - if 12784 <= code <= 13311: res = _charnames_12784[code-12784] - if 19904 <= code <= 19967: res = _charnames_19904[code-19904] - if 40960 <= code <= 42182: res = _charnames_40960[code-40960] - if 42192 <= code <= 42539: res = _charnames_42192[code-42192] - if 42560 <= code <= 42611: res = _charnames_42560[code-42560] - if 42620 <= code <= 42647: res = _charnames_42620[code-42620] - if 42656 <= code <= 42743: res = _charnames_42656[code-42656] - if 42752 <= code <= 42892: res = _charnames_42752[code-42752] - if 43003 <= code <= 43127: res = _charnames_43003[code-43003] - if 43136 <= code <= 43204: res = _charnames_43136[code-43136] - if 43214 <= code <= 43347: res = _charnames_43214[code-43214] - if 43359 <= code <= 43487: res = _charnames_43359[code-43359] - if 43520 <= code <= 43574: res = _charnames_43520[code-43520] - if 43584 <= code <= 43714: res = _charnames_43584[code-43584] - if code == 43739: res = 12781 - if code == 43740: res = 12784 - if code == 43741: res = 12783 - if code == 43742: res = 12785 - if code == 43743: res = 12782 - if 43968 <= code <= 44025: res = _charnames_43968[code-43968] - if 55216 <= code <= 55291: res = _charnames_55216[code-55216] - if 63744 <= code <= 64217: res = _charnames_63744[code-63744] - if 64256 <= code <= 64262: res = _charnames_64256[code-64256] - if 64275 <= code <= 64433: res = _charnames_64275[code-64275] - if 64467 <= code <= 64831: res = _charnames_64467[code-64467] - if 64848 <= code <= 64967: res = _charnames_64848[code-64848] - if 65008 <= code <= 65062: res = _charnames_65008[code-65008] - if 65072 <= code <= 65518: res = _charnames_65072[code-65072] - if 65529 <= code <= 65629: res = _charnames_65529[code-65529] - if 65664 <= code <= 65947: res = _charnames_65664[code-65664] - if 66000 <= code <= 66045: res = _charnames_66000[code-66000] - if 66176 <= code <= 66256: res = _charnames_66176[code-66176] - if 66304 <= code <= 66339: res = _charnames_66304[code-66304] - if 66352 <= code <= 66378: res = _charnames_66352[code-66352] - if 66432 <= code <= 66517: res = _charnames_66432[code-66432] - if 66560 <= code <= 66729: res = _charnames_66560[code-66560] - if 67584 <= code <= 67679: res = _charnames_67584[code-67584] - if 67840 <= code <= 67903: res = _charnames_67840[code-67840] - if 68096 <= code <= 68167: res = _charnames_68096[code-68096] - if 68176 <= code <= 68184: res = _charnames_68176[code-68176] - if 68192 <= code <= 68223: res = _charnames_68192[code-68192] - if 68352 <= code <= 68479: res = _charnames_68352[code-68352] - if 68608 <= code <= 68680: res = _charnames_68608[code-68608] - if 69216 <= code <= 69246: res = _charnames_69216[code-69216] - if 69760 <= code <= 69825: res = _charnames_69760[code-69760] - if 73728 <= code <= 74606: res = _charnames_73728[code-73728] - if 74752 <= code <= 74850: res = _charnames_74752[code-74752] - if code == 74864: res = 2527 - if code == 74865: res = 2526 - if code == 74866: res = 2525 - if code == 74867: res = 2524 - if 77824 <= code <= 78894: res = _charnames_77824[code-77824] - if 118784 <= code <= 119029: res = _charnames_118784[code-118784] - if 119040 <= code <= 119261: res = _charnames_119040[code-119040] - if 119296 <= code <= 119365: res = _charnames_119296[code-119296] - if 119552 <= code <= 119638: res = _charnames_119552[code-119552] - if 119648 <= code <= 119665: res = _charnames_119648[code-119648] - if 119808 <= code <= 120831: res = _charnames_119808[code-119808] - if 126976 <= code <= 127123: res = _charnames_126976[code-126976] - if 127232 <= code <= 127281: res = _charnames_127232[code-127232] - if 127293 <= code <= 127310: res = _charnames_127293[code-127293] - if code == 127319: res = 27110 - if code == 127327: res = 27109 - if 127353 <= code <= 127359: res = _charnames_127353[code-127353] - if 127370 <= code <= 127376: res = _charnames_127370[code-127370] - if code == 127488: res = 19008 - if 127504 <= code <= 127537: res = _charnames_127504[code-127504] - if 127552 <= code <= 127560: res = _charnames_127552[code-127552] - if 194560 <= code <= 195101: res = _charnames_194560[code-194560] - if code == 917505: res = 9289 - if 917536 <= code <= 917631: res = _charnames_917536[code-917536] - if 917760 <= code <= 917999: res = _charnames_917760[code-917760] - if 983040 <= code <= 983050: res = _charnames_983040[code-983040] - if 983552 <= code <= 983945: res = _charnames_983552[code-983552] + elif 160 <= code <= 1317: res = _charnames_160[code-160] + elif 1329 <= code <= 1479: res = _charnames_1329[code-1329] + elif 1488 <= code <= 1524: res = _charnames_1488[code-1488] + elif 1536 <= code <= 1969: res = _charnames_1536[code-1536] + elif 1984 <= code <= 2110: res = _charnames_1984[code-1984] + elif 2304 <= code <= 2510: res = _charnames_2304[code-2304] + elif 2519 <= code <= 2641: res = _charnames_2519[code-2519] + elif code == 2649: res = 22369 + elif code == 2650: res = 22374 + elif code == 2651: res = 22402 + elif code == 2652: res = 22383 + elif code == 2654: res = 22408 + elif 2662 <= code <= 2677: res = _charnames_2662[code-2662] + elif 2689 <= code <= 2768: res = _charnames_2689[code-2689] + elif 2784 <= code <= 2801: res = _charnames_2784[code-2784] + elif 2817 <= code <= 2893: res = _charnames_2817[code-2817] + elif 2902 <= code <= 2929: res = _charnames_2902[code-2902] + elif 2946 <= code <= 3031: res = _charnames_2946[code-2946] + elif 3046 <= code <= 3149: res = _charnames_3046[code-3046] + elif 3157 <= code <= 3183: res = _charnames_3157[code-3157] + elif 3192 <= code <= 3277: res = _charnames_3192[code-3192] + elif code == 3285: res = 23172 + elif code == 3286: res = 23215 + elif 3294 <= code <= 3314: res = _charnames_3294[code-3294] + elif 3330 <= code <= 3405: res = _charnames_3330[code-3330] + elif code == 3415: res = 6443 + elif 3424 <= code <= 3551: res = _charnames_3424[code-3424] + elif code == 3570: res = 19930 + elif code == 3571: res = 19929 + elif code == 3572: res = 19944 + elif 3585 <= code <= 3675: res = _charnames_3585[code-3585] + elif 3713 <= code <= 3805: res = _charnames_3713[code-3713] + elif 3840 <= code <= 4056: res = _charnames_3840[code-3840] + elif 4096 <= code <= 4293: res = _charnames_4096[code-4096] + elif 4304 <= code <= 5108: res = _charnames_4304[code-4304] + elif 5120 <= code <= 5872: res = _charnames_5120[code-5120] + elif 5888 <= code <= 5908: res = _charnames_5888[code-5888] + elif 5920 <= code <= 5942: res = _charnames_5920[code-5920] + elif 5952 <= code <= 5971: res = _charnames_5952[code-5952] + elif 5984 <= code <= 6003: res = _charnames_5984[code-5984] + elif 6016 <= code <= 6263: res = _charnames_6016[code-6016] + elif 6272 <= code <= 6389: res = _charnames_6272[code-6272] + elif 6400 <= code <= 6516: res = _charnames_6400[code-6400] + elif 6528 <= code <= 6829: res = _charnames_6528[code-6528] + elif 6912 <= code <= 7097: res = _charnames_6912[code-6912] + elif 7168 <= code <= 7295: res = _charnames_7168[code-7168] + elif 7376 <= code <= 7410: res = _charnames_7376[code-7376] + elif 7424 <= code <= 7654: res = _charnames_7424[code-7424] + elif 7677 <= code <= 8340: res = _charnames_7677[code-7677] + elif 8352 <= code <= 8376: res = _charnames_8352[code-8352] + elif 8400 <= code <= 8432: res = _charnames_8400[code-8400] + elif 8448 <= code <= 9192: res = _charnames_8448[code-8448] + elif 9216 <= code <= 9254: res = _charnames_9216[code-9216] + elif 9280 <= code <= 9290: res = _charnames_9280[code-9280] + elif 9312 <= code <= 11097: res = _charnames_9312[code-9312] + elif 11264 <= code <= 11505: res = _charnames_11264[code-11264] + elif 11513 <= code <= 11557: res = _charnames_11513[code-11513] + elif 11568 <= code <= 11621: res = _charnames_11568[code-11568] + elif code == 11631: res = 13355 + elif 11648 <= code <= 11670: res = _charnames_11648[code-11648] + elif 11680 <= code <= 11825: res = _charnames_11680[code-11680] + elif 11904 <= code <= 12019: res = _charnames_11904[code-11904] + elif 12032 <= code <= 12245: res = _charnames_12032[code-12032] + elif 12272 <= code <= 12727: res = _charnames_12272[code-12272] + elif 12736 <= code <= 12771: res = _charnames_12736[code-12736] + elif 12784 <= code <= 13311: res = _charnames_12784[code-12784] + elif 19904 <= code <= 19967: res = _charnames_19904[code-19904] + elif 40960 <= code <= 42182: res = _charnames_40960[code-40960] + elif 42192 <= code <= 42539: res = _charnames_42192[code-42192] + elif 42560 <= code <= 42611: res = _charnames_42560[code-42560] + elif 42620 <= code <= 42647: res = _charnames_42620[code-42620] + elif 42656 <= code <= 42743: res = _charnames_42656[code-42656] + elif 42752 <= code <= 42892: res = _charnames_42752[code-42752] + elif 43003 <= code <= 43127: res = _charnames_43003[code-43003] + elif 43136 <= code <= 43204: res = _charnames_43136[code-43136] + elif 43214 <= code <= 43347: res = _charnames_43214[code-43214] + elif 43359 <= code <= 43487: res = _charnames_43359[code-43359] + elif 43520 <= code <= 43574: res = _charnames_43520[code-43520] + elif 43584 <= code <= 43714: res = _charnames_43584[code-43584] + elif code == 43739: res = 12781 + elif code == 43740: res = 12784 + elif code == 43741: res = 12783 + elif code == 43742: res = 12785 + elif code == 43743: res = 12782 + elif 43968 <= code <= 44025: res = _charnames_43968[code-43968] + elif 55216 <= code <= 55291: res = _charnames_55216[code-55216] + elif 63744 <= code <= 64217: res = _charnames_63744[code-63744] + elif 64256 <= code <= 64262: res = _charnames_64256[code-64256] + elif 64275 <= code <= 64433: res = _charnames_64275[code-64275] + elif 64467 <= code <= 64831: res = _charnames_64467[code-64467] + elif 64848 <= code <= 64967: res = _charnames_64848[code-64848] + elif 65008 <= code <= 65062: res = _charnames_65008[code-65008] + elif 65072 <= code <= 65518: res = _charnames_65072[code-65072] + elif 65529 <= code <= 65629: res = _charnames_65529[code-65529] + elif 65664 <= code <= 65947: res = _charnames_65664[code-65664] + elif 66000 <= code <= 66045: res = _charnames_66000[code-66000] + elif 66176 <= code <= 66256: res = _charnames_66176[code-66176] + elif 66304 <= code <= 66339: res = _charnames_66304[code-66304] + elif 66352 <= code <= 66378: res = _charnames_66352[code-66352] + elif 66432 <= code <= 66517: res = _charnames_66432[code-66432] + elif 66560 <= code <= 66729: res = _charnames_66560[code-66560] + elif 67584 <= code <= 67679: res = _charnames_67584[code-67584] + elif 67840 <= code <= 67903: res = _charnames_67840[code-67840] + elif 68096 <= code <= 68167: res = _charnames_68096[code-68096] + elif 68176 <= code <= 68184: res = _charnames_68176[code-68176] + elif 68192 <= code <= 68223: res = _charnames_68192[code-68192] + elif 68352 <= code <= 68479: res = _charnames_68352[code-68352] + elif 68608 <= code <= 68680: res = _charnames_68608[code-68608] + elif 69216 <= code <= 69246: res = _charnames_69216[code-69216] + elif 69760 <= code <= 69825: res = _charnames_69760[code-69760] + elif 73728 <= code <= 74606: res = _charnames_73728[code-73728] + elif 74752 <= code <= 74850: res = _charnames_74752[code-74752] + elif code == 74864: res = 2527 + elif code == 74865: res = 2526 + elif code == 74866: res = 2525 + elif code == 74867: res = 2524 + elif 77824 <= code <= 78894: res = _charnames_77824[code-77824] + elif 118784 <= code <= 119029: res = _charnames_118784[code-118784] + elif 119040 <= code <= 119261: res = _charnames_119040[code-119040] + elif 119296 <= code <= 119365: res = _charnames_119296[code-119296] + elif 119552 <= code <= 119638: res = _charnames_119552[code-119552] + elif 119648 <= code <= 119665: res = _charnames_119648[code-119648] + elif 119808 <= code <= 120831: res = _charnames_119808[code-119808] + elif 126976 <= code <= 127123: res = _charnames_126976[code-126976] + elif 127232 <= code <= 127281: res = _charnames_127232[code-127232] + elif 127293 <= code <= 127310: res = _charnames_127293[code-127293] + elif code == 127319: res = 27110 + elif code == 127327: res = 27109 + elif 127353 <= code <= 127359: res = _charnames_127353[code-127353] + elif 127370 <= code <= 127376: res = _charnames_127370[code-127370] + elif code == 127488: res = 19008 + elif 127504 <= code <= 127537: res = _charnames_127504[code-127504] + elif 127552 <= code <= 127560: res = _charnames_127552[code-127552] + elif 194560 <= code <= 195101: res = _charnames_194560[code-194560] + elif code == 917505: res = 9289 + elif 917536 <= code <= 917631: res = _charnames_917536[code-917536] + elif 917760 <= code <= 917999: res = _charnames_917760[code-917760] + elif 983040 <= code <= 983050: res = _charnames_983040[code-983040] + elif 983552 <= code <= 983945: res = _charnames_983552[code-983552] if res == -1: raise KeyError, code return name_of_node(res) +# end output from build_compression_tree +#____________________________________________________________ # the following dictionary is used by modules that take this as a base +# only used by generate_unicodedb, not after translation _orig_names = { 'AC CURRENT': 9190, 'ACCOUNT OF': 8448, @@ -136813,7 +136823,7 @@ code = trie_lookup(name) else: try: - code = _code_by_name[name] + code = trie_lookup(name) except KeyError: if name not in _code_by_name_corrected: code = base_mod.trie_lookup(name) @@ -136842,7 +136852,7 @@ return lookup_charcode(code) else: try: - return _names[code] + return lookup_charcode(code) except KeyError: if code not in _names_corrected: return base_mod.lookup_charcode(code) diff --git a/rpython/rlib/unicodedata/unicodedb_6_0_0.py b/rpython/rlib/unicodedata/unicodedb_6_0_0.py --- a/rpython/rlib/unicodedata/unicodedb_6_0_0.py +++ b/rpython/rlib/unicodedata/unicodedb_6_0_0.py @@ -8,4460 +8,13026 @@ import unicodedb_5_2_0 as base_mod version = '6.0.0' -_names = { -128673: 'AERIAL TRAMWAY', -9200: 'ALARM CLOCK', -128769: 'ALCHEMICAL SYMBOL FOR AIR', -128874: 'ALCHEMICAL SYMBOL FOR ALEMBIC', -128822: 'ALCHEMICAL SYMBOL FOR ALKALI', -128823: 'ALCHEMICAL SYMBOL FOR ALKALI-2', -128837: 'ALCHEMICAL SYMBOL FOR ALUM', -128859: 'ALCHEMICAL SYMBOL FOR AMALGAM', -128811: 'ALCHEMICAL SYMBOL FOR ANTIMONY ORE', -128774: 'ALCHEMICAL SYMBOL FOR AQUA REGIA', -128775: 'ALCHEMICAL SYMBOL FOR AQUA REGIA-2', -128776: 'ALCHEMICAL SYMBOL FOR AQUA VITAE', -128777: 'ALCHEMICAL SYMBOL FOR AQUA VITAE-2', -128773: 'ALCHEMICAL SYMBOL FOR AQUAFORTIS', -128826: 'ALCHEMICAL SYMBOL FOR ARSENIC', -128855: 'ALCHEMICAL SYMBOL FOR ASHES', -128829: 'ALCHEMICAL SYMBOL FOR AURIPIGMENT', -128875: 'ALCHEMICAL SYMBOL FOR BATH OF MARY', -128876: 'ALCHEMICAL SYMBOL FOR BATH OF VAPOURS', -128830: 'ALCHEMICAL SYMBOL FOR BISMUTH ORE', -128783: 'ALCHEMICAL SYMBOL FOR BLACK SULFUR', -128834: 'ALCHEMICAL SYMBOL FOR BORAX', -128835: 'ALCHEMICAL SYMBOL FOR BORAX-2', -128836: 'ALCHEMICAL SYMBOL FOR BORAX-3', -128857: 'ALCHEMICAL SYMBOL FOR BRICK', -128848: 'ALCHEMICAL SYMBOL FOR CADUCEUS', -128844: 'ALCHEMICAL SYMBOL FOR CALX', -128846: 'ALCHEMICAL SYMBOL FOR CAPUT MORTUUM', -128787: 'ALCHEMICAL SYMBOL FOR CINNABAR', -128805: 'ALCHEMICAL SYMBOL FOR COPPER ANTIMONIATE', -128800: 'ALCHEMICAL SYMBOL FOR COPPER ORE', -128803: 'ALCHEMICAL SYMBOL FOR CROCUS OF COPPER', -128804: 'ALCHEMICAL SYMBOL FOR CROCUS OF COPPER-2', -128798: 'ALCHEMICAL SYMBOL FOR CROCUS OF IRON', -128869: 'ALCHEMICAL SYMBOL FOR CRUCIBLE', -128870: 'ALCHEMICAL SYMBOL FOR CRUCIBLE-2', -128871: 'ALCHEMICAL SYMBOL FOR CRUCIBLE-3', -128872: 'ALCHEMICAL SYMBOL FOR CRUCIBLE-4', -128873: 'ALCHEMICAL SYMBOL FOR CRUCIBLE-5', -128880: 'ALCHEMICAL SYMBOL FOR DAY-NIGHT', -128865: 'ALCHEMICAL SYMBOL FOR DISSOLVE', -128866: 'ALCHEMICAL SYMBOL FOR DISSOLVE-2', -128864: 'ALCHEMICAL SYMBOL FOR DISTILL', -128771: 'ALCHEMICAL SYMBOL FOR EARTH', -128770: 'ALCHEMICAL SYMBOL FOR FIRE', -128794: 'ALCHEMICAL SYMBOL FOR GOLD', -128841: 'ALCHEMICAL SYMBOL FOR GUM', -128882: 'ALCHEMICAL SYMBOL FOR HALF DRAM', -128883: 'ALCHEMICAL SYMBOL FOR HALF OUNCE', -128854: 'ALCHEMICAL SYMBOL FOR HORSE DUNG', -128878: 'ALCHEMICAL SYMBOL FOR HOUR', -128796: 'ALCHEMICAL SYMBOL FOR IRON ORE', -128797: 'ALCHEMICAL SYMBOL FOR IRON ORE-2', -128801: 'ALCHEMICAL SYMBOL FOR IRON-COPPER ORE', -128810: 'ALCHEMICAL SYMBOL FOR LEAD ORE', -128851: 'ALCHEMICAL SYMBOL FOR LODESTONE', -128824: 'ALCHEMICAL SYMBOL FOR MARCASITE', -128784: 'ALCHEMICAL SYMBOL FOR MERCURY SUBLIMATE', -128785: 'ALCHEMICAL SYMBOL FOR MERCURY SUBLIMATE-2', -128786: 'ALCHEMICAL SYMBOL FOR MERCURY SUBLIMATE-3', -128881: 'ALCHEMICAL SYMBOL FOR MONTH', -128879: 'ALCHEMICAL SYMBOL FOR NIGHT', -128789: 'ALCHEMICAL SYMBOL FOR NITRE', -128838: 'ALCHEMICAL SYMBOL FOR OIL', -128782: 'ALCHEMICAL SYMBOL FOR PHILOSOPHERS SULFUR', -128856: 'ALCHEMICAL SYMBOL FOR POT ASHES', -128843: 'ALCHEMICAL SYMBOL FOR POWDER', -128858: 'ALCHEMICAL SYMBOL FOR POWDERED BRICK', -128863: 'ALCHEMICAL SYMBOL FOR PRECIPITATE', -128867: 'ALCHEMICAL SYMBOL FOR PURIFY', -128868: 'ALCHEMICAL SYMBOL FOR PUTREFACTION', -128833: 'ALCHEMICAL SYMBOL FOR QUICK LIME', -128768: 'ALCHEMICAL SYMBOL FOR QUINTESSENCE', -128827: 'ALCHEMICAL SYMBOL FOR REALGAR', -128828: 'ALCHEMICAL SYMBOL FOR REALGAR-2', -128818: 'ALCHEMICAL SYMBOL FOR REGULUS', -128816: 'ALCHEMICAL SYMBOL FOR REGULUS OF ANTIMONY', -128817: 'ALCHEMICAL SYMBOL FOR REGULUS OF ANTIMONY-2', -128799: 'ALCHEMICAL SYMBOL FOR REGULUS OF IRON', -128819: 'ALCHEMICAL SYMBOL FOR REGULUS-2', -128820: 'ALCHEMICAL SYMBOL FOR REGULUS-3', -128821: 'ALCHEMICAL SYMBOL FOR REGULUS-4', -128877: 'ALCHEMICAL SYMBOL FOR RETORT', -128792: 'ALCHEMICAL SYMBOL FOR ROCK SALT', -128793: 'ALCHEMICAL SYMBOL FOR ROCK SALT-2', -128825: 'ALCHEMICAL SYMBOL FOR SAL-AMMONIAC', -128788: 'ALCHEMICAL SYMBOL FOR SALT', -128813: 'ALCHEMICAL SYMBOL FOR SALT OF ANTIMONY', -128806: 'ALCHEMICAL SYMBOL FOR SALT OF COPPER ANTIMONIATE', -128847: 'ALCHEMICAL SYMBOL FOR SCEPTER OF JOVE', -128795: 'ALCHEMICAL SYMBOL FOR SILVER', -128852: 'ALCHEMICAL SYMBOL FOR SOAP', -128839: 'ALCHEMICAL SYMBOL FOR SPIRIT', -128850: 'ALCHEMICAL SYMBOL FOR STARRED TRIDENT', -128860: 'ALCHEMICAL SYMBOL FOR STRATUM SUPER STRATUM', -128861: 'ALCHEMICAL SYMBOL FOR STRATUM SUPER STRATUM-2', -128812: 'ALCHEMICAL SYMBOL FOR SUBLIMATE OF ANTIMONY', -128802: 'ALCHEMICAL SYMBOL FOR SUBLIMATE OF COPPER', -128814: 'ALCHEMICAL SYMBOL FOR SUBLIMATE OF SALT OF ANTIMONY', -128807: 'ALCHEMICAL SYMBOL FOR SUBLIMATE OF SALT OF COPPER', -128862: 'ALCHEMICAL SYMBOL FOR SUBLIMATION', -128781: 'ALCHEMICAL SYMBOL FOR SULFUR', -128831: 'ALCHEMICAL SYMBOL FOR TARTAR', -128832: 'ALCHEMICAL SYMBOL FOR TARTAR-2', -128809: 'ALCHEMICAL SYMBOL FOR TIN ORE', -128840: 'ALCHEMICAL SYMBOL FOR TINCTURE', -128849: 'ALCHEMICAL SYMBOL FOR TRIDENT', -128845: 'ALCHEMICAL SYMBOL FOR TUTTY', -128853: 'ALCHEMICAL SYMBOL FOR URINE', -128808: 'ALCHEMICAL SYMBOL FOR VERDIGRIS', -128778: 'ALCHEMICAL SYMBOL FOR VINEGAR', -128815: 'ALCHEMICAL SYMBOL FOR VINEGAR OF ANTIMONY', -128779: 'ALCHEMICAL SYMBOL FOR VINEGAR-2', -128780: 'ALCHEMICAL SYMBOL FOR VINEGAR-3', -128790: 'ALCHEMICAL SYMBOL FOR VITRIOL', -128791: 'ALCHEMICAL SYMBOL FOR VITRIOL-2', -128772: 'ALCHEMICAL SYMBOL FOR WATER', -128842: 'ALCHEMICAL SYMBOL FOR WAX', -128126: 'ALIEN MONSTER', -128657: 'AMBULANCE', -127944: 'AMERICAN FOOTBALL', -128162: 'ANGER SYMBOL', -128544: 'ANGRY FACE', -128028: 'ANT', -128246: 'ANTENNA WITH BARS', -128260: 'ANTICLOCKWISE DOWNWARDS AND UPWARDS OPEN CIRCLE ARROWS', -1568: 'ARABIC LETTER KASHMIRI YEH', -64434: 'ARABIC SYMBOL DOT ABOVE', -64435: 'ARABIC SYMBOL DOT BELOW', -64444: 'ARABIC SYMBOL DOUBLE VERTICAL BAR BELOW', -64442: 'ARABIC SYMBOL FOUR DOTS ABOVE', -64443: 'ARABIC SYMBOL FOUR DOTS BELOW', -64447: 'ARABIC SYMBOL RING', -64448: 'ARABIC SYMBOL SMALL TAH ABOVE', -64449: 'ARABIC SYMBOL SMALL TAH BELOW', -64438: 'ARABIC SYMBOL THREE DOTS ABOVE', -64439: 'ARABIC SYMBOL THREE DOTS BELOW', -64440: 'ARABIC SYMBOL THREE DOTS POINTING DOWNWARDS ABOVE', -64441: 'ARABIC SYMBOL THREE DOTS POINTING DOWNWARDS BELOW', -64436: 'ARABIC SYMBOL TWO DOTS ABOVE', -64437: 'ARABIC SYMBOL TWO DOTS BELOW', -64445: 'ARABIC SYMBOL TWO DOTS VERTICALLY ABOVE', -64446: 'ARABIC SYMBOL TWO DOTS VERTICALLY BELOW', -1631: 'ARABIC WAVY HAMZA BELOW', -128667: 'ARTICULATED LORRY', -127912: 'ARTIST PALETTE', -128562: 'ASTONISHED FACE', -9954: 'ASTRONOMICAL SYMBOL FOR URANUS', -128095: 'ATHLETIC SHOE', -127814: 'AUBERGINE', -127975: 'AUTOMATED TELLER MACHINE', -128663: 'AUTOMOBILE', -128118: 'BABY', -128124: 'BABY ANGEL', -127868: 'BABY BOTTLE', -128036: 'BABY CHICK', -128700: 'BABY SYMBOL', -128281: 'BACK WITH LEFTWARDS ARROW ABOVE', -128043: 'BACTRIAN CAMEL', -128708: 'BAGGAGE CLAIM', -127880: 'BALLOON', -92217: 'BAMUM LETTER PHASE-A FIRI', -92161: 'BAMUM LETTER PHASE-A GBIEE FON', -92193: 'BAMUM LETTER PHASE-A GHEUAEGHEUAE', -92188: 'BAMUM LETTER PHASE-A GHEUAERAE', -92199: 'BAMUM LETTER PHASE-A KAFA', -92240: 'BAMUM LETTER PHASE-A KAQ', -92211: 'BAMUM LETTER PHASE-A KET', -92179: 'BAMUM LETTER PHASE-A KEUKEUTNDA', -92219: 'BAMUM LETTER PHASE-A KPOQ', -92213: 'BAMUM LETTER PHASE-A KUOQ', -92183: 'BAMUM LETTER PHASE-A LAPAQ', -92184: 'BAMUM LETTER PHASE-A LET KUT', -92216: 'BAMUM LETTER PHASE-A LOMMAE', -92243: 'BAMUM LETTER PHASE-A LU', -92207: 'BAMUM LETTER PHASE-A LUAEP', -92186: 'BAMUM LETTER PHASE-A MAEKEUP', -92238: 'BAMUM LETTER PHASE-A MAEM', -92171: 'BAMUM LETTER PHASE-A MAEMBGBIEE', -92203: 'BAMUM LETTER PHASE-A MAEMKPEN', -92174: 'BAMUM LETTER PHASE-A MAEMVEUX', -92210: 'BAMUM LETTER PHASE-A MAENYI', -92230: 'BAMUM LETTER PHASE-A MAESI', -92175: 'BAMUM LETTER PHASE-A MANSUAE', -92221: 'BAMUM LETTER PHASE-A MAP PIEET', -92232: 'BAMUM LETTER PHASE-A MBANYI', -92246: 'BAMUM LETTER PHASE-A MBAQ', -92197: 'BAMUM LETTER PHASE-A MEUNJOMNDEUQ', -92196: 'BAMUM LETTER PHASE-A MGBASA', -92190: 'BAMUM LETTER PHASE-A MON NGGEUAET', -92214: 'BAMUM LETTER PHASE-A MOOMEUT', -92198: 'BAMUM LETTER PHASE-A MOOMPUQ', -92176: 'BAMUM LETTER PHASE-A MVEUAENGAM', -92164: 'BAMUM LETTER PHASE-A NAA MFON', -92245: 'BAMUM LETTER PHASE-A NAQ', -92201: 'BAMUM LETTER PHASE-A NDA LEERAEWA', -92212: 'BAMUM LETTER PHASE-A NDAANGGEUAET', -92244: 'BAMUM LETTER PHASE-A NEN', -92173: 'BAMUM LETTER PHASE-A NGANGU', -92229: 'BAMUM LETTER PHASE-A NGGEN', -92160: 'BAMUM LETTER PHASE-A NGKUE MFON', -92182: 'BAMUM LETTER PHASE-A NGKUENZEUM', -92204: 'BAMUM LETTER PHASE-A NIKA', -92231: 'BAMUM LETTER PHASE-A NJAM', -92227: 'BAMUM LETTER PHASE-A NKAARAE', -92180: 'BAMUM LETTER PHASE-A NKINDI', -92241: 'BAMUM LETTER PHASE-A NSHA', -92237: 'BAMUM LETTER PHASE-A NSHIEE', -92223: 'BAMUM LETTER PHASE-A NTAP', -92185: 'BAMUM LETTER PHASE-A NTAP MFAA', -92194: 'BAMUM LETTER PHASE-A NTAP NTAA', -92178: 'BAMUM LETTER PHASE-A NTOQPEN', -92233: 'BAMUM LETTER PHASE-A NYET', -92239: 'BAMUM LETTER PHASE-A NYI', -92225: 'BAMUM LETTER PHASE-A NYIT MONGKEUAEQ', -92167: 'BAMUM LETTER PHASE-A NZA MFON', -92191: 'BAMUM LETTER PHASE-A NZUN MEUT', -92200: 'BAMUM LETTER PHASE-A PA LEERAEWA', -92236: 'BAMUM LETTER PHASE-A PAAM', -92226: 'BAMUM LETTER PHASE-A PAARAE', -92189: 'BAMUM LETTER PHASE-A PAMSHAE', -92187: 'BAMUM LETTER PHASE-A PASHAE', -92202: 'BAMUM LETTER PHASE-A PET', -92163: 'BAMUM LETTER PHASE-A PON MFON PIPAEMBA', -92162: 'BAMUM LETTER PHASE-A PON MFON PIPAEMGBIEE', -92170: 'BAMUM LETTER PHASE-A PON PA NJI PIPAEMBA', -92169: 'BAMUM LETTER PHASE-A PON PA NJI PIPAEMGBIEE', -92205: 'BAMUM LETTER PHASE-A PUP', -92218: 'BAMUM LETTER PHASE-A ROM', -92177: 'BAMUM LETTER PHASE-A SEUNYAM', -92168: 'BAMUM LETTER PHASE-A SHINDA PA NJI', -92222: 'BAMUM LETTER PHASE-A SHIRAE', -92224: 'BAMUM LETTER PHASE-A SHOQ NSHUT YUM', -92165: 'BAMUM LETTER PHASE-A SHUENSHUET', -92215: 'BAMUM LETTER PHASE-A SHUM', -92195: 'BAMUM LETTER PHASE-A SISA', -92208: 'BAMUM LETTER PHASE-A SONJAM', -92220: 'BAMUM LETTER PHASE-A SOQ', -92235: 'BAMUM LETTER PHASE-A SOT', -92181: 'BAMUM LETTER PHASE-A SUU', -92234: 'BAMUM LETTER PHASE-A TEUAEN', -92209: 'BAMUM LETTER PHASE-A TEUTEUWEN', -92166: 'BAMUM LETTER PHASE-A TITA MFON', -92172: 'BAMUM LETTER PHASE-A TU MAEMBA', -92206: 'BAMUM LETTER PHASE-A TUAEP', -92192: 'BAMUM LETTER PHASE-A U YUQ NAE', -92228: 'BAMUM LETTER PHASE-A UNKNOWN', -92242: 'BAMUM LETTER PHASE-A VEE', -92293: 'BAMUM LETTER PHASE-B FEE', -92291: 'BAMUM LETTER PHASE-B FEUX', -92271: 'BAMUM LETTER PHASE-B GHEUGHEN', -92262: 'BAMUM LETTER PHASE-B GHEUGHEUAEM', -92256: 'BAMUM LETTER PHASE-B KAM', -92294: 'BAMUM LETTER PHASE-B KEUAEM', -92270: 'BAMUM LETTER PHASE-B KEUPUQ', -92272: 'BAMUM LETTER PHASE-B KEUYEUX', -92253: 'BAMUM LETTER PHASE-B KIEEM', -92301: 'BAMUM LETTER PHASE-B KIQ', -92273: 'BAMUM LETTER PHASE-B LAANAE', -92259: 'BAMUM LETTER PHASE-B LAM NSHUT NYAM', -92297: 'BAMUM LETTER PHASE-B LET', -92251: 'BAMUM LETTER PHASE-B LOM NTEUM', -92300: 'BAMUM LETTER PHASE-B MA', -92295: 'BAMUM LETTER PHASE-B MA NJEUAENA', -92296: 'BAMUM LETTER PHASE-B MA NJUQA', -92252: 'BAMUM LETTER PHASE-B MBA MAELEE', -92255: 'BAMUM LETTER PHASE-B MBAARAE', -92286: 'BAMUM LETTER PHASE-B MBEURI', -92268: 'BAMUM LETTER PHASE-B MBIT MBAAKET', -92292: 'BAMUM LETTER PHASE-B MBUOQ', -92281: 'BAMUM LETTER PHASE-B MEUQ', -92290: 'BAMUM LETTER PHASE-B MEUT NGGEET', -92284: 'BAMUM LETTER PHASE-B MFIYAQ', -92267: 'BAMUM LETTER PHASE-B MFON TEUAEQ', -92287: 'BAMUM LETTER PHASE-B MONTIEEN', -92261: 'BAMUM LETTER PHASE-B NDU NJAA', -92298: 'BAMUM LETTER PHASE-B NGGAAM', -92277: 'BAMUM LETTER PHASE-B NGGEU MBU', -92282: 'BAMUM LETTER PHASE-B NGGUOQ', -92283: 'BAMUM LETTER PHASE-B NGGUOQ LARGE', -92276: 'BAMUM LETTER PHASE-B NGKINDI MVOP', -92302: 'BAMUM LETTER PHASE-B NGOM', -92299: 'BAMUM LETTER PHASE-B NSEN', -92247: 'BAMUM LETTER PHASE-B NSHUET', -92260: 'BAMUM LETTER PHASE-B NTIEE SHEUOQ', -92288: 'BAMUM LETTER PHASE-B NYAEMAE', -92269: 'BAMUM LETTER PHASE-B NYI NTEUM', -92274: 'BAMUM LETTER PHASE-B PARUM', -92257: 'BAMUM LETTER PHASE-B PEESHI', -92263: 'BAMUM LETTER PHASE-B PIT', -92289: 'BAMUM LETTER PHASE-B PUNGAAM', -92279: 'BAMUM LETTER PHASE-B SAKEUAE', -92250: 'BAMUM LETTER PHASE-B SET TU', -92265: 'BAMUM LETTER PHASE-B SHET NJAQ', -92266: 'BAMUM LETTER PHASE-B SHEUAEQTU', -92249: 'BAMUM LETTER PHASE-B SIEE', -92285: 'BAMUM LETTER PHASE-B SUE', -92280: 'BAMUM LETTER PHASE-B TAAM', -92248: 'BAMUM LETTER PHASE-B TU MAEMGBIEE', -92264: 'BAMUM LETTER PHASE-B TU NSIEE', -92275: 'BAMUM LETTER PHASE-B VEUM', -92278: 'BAMUM LETTER PHASE-B WUAET', -92258: 'BAMUM LETTER PHASE-B YAFU LEERAEWA', -92254: 'BAMUM LETTER PHASE-B YEURAE', -92352: 'BAMUM LETTER PHASE-C BUNG', -92348: 'BAMUM LETTER PHASE-C FUE', -92312: 'BAMUM LETTER PHASE-C GBAYI', -92320: 'BAMUM LETTER PHASE-C GHAP', -92310: 'BAMUM LETTER PHASE-C GHARAE', -92329: 'BAMUM LETTER PHASE-C KAA', -92394: 'BAMUM LETTER PHASE-C KEN FATIGUE', -92393: 'BAMUM LETTER PHASE-C KEN LAW', -92361: 'BAMUM LETTER PHASE-C KET', -92321: 'BAMUM LETTER PHASE-C KEUKAQ', -92386: 'BAMUM LETTER PHASE-C KEUM', -92384: 'BAMUM LETTER PHASE-C KEUSEUX', -92319: 'BAMUM LETTER PHASE-C KEUSHEUAEP', -92328: 'BAMUM LETTER PHASE-C KPARAQ', -92373: 'BAMUM LETTER PHASE-C KUOP NKAARAE', -92366: 'BAMUM LETTER PHASE-C KUT', -92357: 'BAMUM LETTER PHASE-C LAM', -92342: 'BAMUM LETTER PHASE-C LAP', -92397: 'BAMUM LETTER PHASE-C LIQ', -92365: 'BAMUM LETTER PHASE-C LU', -92339: 'BAMUM LETTER PHASE-C MA KEUAERI', -92376: 'BAMUM LETTER PHASE-C MA NSIEE', -92382: 'BAMUM LETTER PHASE-C MAEMBA', -92363: 'BAMUM LETTER PHASE-C MAESI', -92318: 'BAMUM LETTER PHASE-C MBAA CABBAGE-TREE', -92387: 'BAMUM LETTER PHASE-C MBAA PICKET', -92383: 'BAMUM LETTER PHASE-C MBANYI', -92311: 'BAMUM LETTER PHASE-C MBEEKEET', -92354: 'BAMUM LETTER PHASE-C MBERAE', -92315: 'BAMUM LETTER PHASE-C MBEUM', -92385: 'BAMUM LETTER PHASE-C MBEUX', -92381: 'BAMUM LETTER PHASE-C MBI', -92343: 'BAMUM LETTER PHASE-C MBIRIEEN', -92326: 'BAMUM LETTER PHASE-C MBIT', -92364: 'BAMUM LETTER PHASE-C MBUAEM', -92324: 'BAMUM LETTER PHASE-C MBUE', -92344: 'BAMUM LETTER PHASE-C MGBASAQ', -92390: 'BAMUM LETTER PHASE-C MIEE', -92391: 'BAMUM LETTER PHASE-C MUAE', -92338: 'BAMUM LETTER PHASE-C NANSANAQ', -92396: 'BAMUM LETTER PHASE-C NAQ', -92375: 'BAMUM LETTER PHASE-C NDAM', -92378: 'BAMUM LETTER PHASE-C NDAP', -92308: 'BAMUM LETTER PHASE-C NDEUAEREE', -92349: 'BAMUM LETTER PHASE-C NDEUT', -92331: 'BAMUM LETTER PHASE-C NDIDA', -92317: 'BAMUM LETTER PHASE-C NDOMBU', -92395: 'BAMUM LETTER PHASE-C NGAQ', -92307: 'BAMUM LETTER PHASE-C NGGEN', -92362: 'BAMUM LETTER PHASE-C NGGU', -92336: 'BAMUM LETTER PHASE-C NGGUAEN NYAM', -92370: 'BAMUM LETTER PHASE-C NGGUEET', -92347: 'BAMUM LETTER PHASE-C NGGUM', -92341: 'BAMUM LETTER PHASE-C NGGUON', -92309: 'BAMUM LETTER PHASE-C NGKAQ', -92303: 'BAMUM LETTER PHASE-C NGKUE MAEMBA', -92368: 'BAMUM LETTER PHASE-C NGOM', -92356: 'BAMUM LETTER PHASE-C NJAEM', -92367: 'BAMUM LETTER PHASE-C NJAM', -92360: 'BAMUM LETTER PHASE-C NJEEEE', -92389: 'BAMUM LETTER PHASE-C NJEUX', -92333: 'BAMUM LETTER PHASE-C NJUEQ', -92350: 'BAMUM LETTER PHASE-C NSA', -92325: 'BAMUM LETTER PHASE-C NSEUAEN', -92351: 'BAMUM LETTER PHASE-C NSHAQ', -92371: 'BAMUM LETTER PHASE-C NSOM', -92374: 'BAMUM LETTER PHASE-C NSUN', -92359: 'BAMUM LETTER PHASE-C NSUOT NGOM', -92340: 'BAMUM LETTER PHASE-C NTAA', -92372: 'BAMUM LETTER PHASE-C NTEN', -92345: 'BAMUM LETTER PHASE-C NTEUNGBA', -92314: 'BAMUM LETTER PHASE-C NTU MBIT', -92313: 'BAMUM LETTER PHASE-C NYIR MKPARAQ MEUN', -92304: 'BAMUM LETTER PHASE-C NZA', -92323: 'BAMUM LETTER PHASE-C NZEUM', -92399: 'BAMUM LETTER PHASE-C PEN', -92398: 'BAMUM LETTER PHASE-C PIN', -92316: 'BAMUM LETTER PHASE-C PIRIEEN', -92355: 'BAMUM LETTER PHASE-C RU', -92380: 'BAMUM LETTER PHASE-C SETFON', -92330: 'BAMUM LETTER PHASE-C SEUX', -92392: 'BAMUM LETTER PHASE-C SHIQ', -92379: 'BAMUM LETTER PHASE-C SHUEQ', -92335: 'BAMUM LETTER PHASE-C SUAET', -92332: 'BAMUM LETTER PHASE-C TAASHAE', -92400: 'BAMUM LETTER PHASE-C TET', -92346: 'BAMUM LETTER PHASE-C TEUTEUX', -92334: 'BAMUM LETTER PHASE-C TITA YUE', -92358: 'BAMUM LETTER PHASE-C TITUAEP', -92353: 'BAMUM LETTER PHASE-C VEUAEPEN', -92337: 'BAMUM LETTER PHASE-C VEUX', -92306: 'BAMUM LETTER PHASE-C WANGKUOQ', -92369: 'BAMUM LETTER PHASE-C WUP', -92377: 'BAMUM LETTER PHASE-C YAA', -92327: 'BAMUM LETTER PHASE-C YEUQ', -92322: 'BAMUM LETTER PHASE-C YU MUOMAE', -92305: 'BAMUM LETTER PHASE-C YUM', -92388: 'BAMUM LETTER PHASE-C YUWOQ', -92517: 'BAMUM LETTER PHASE-D FAA', -92436: 'BAMUM LETTER PHASE-D FEUFEUAET', -92434: 'BAMUM LETTER PHASE-D GHAA', -92488: 'BAMUM LETTER PHASE-D GHEUAE', -92420: 'BAMUM LETTER PHASE-D KET', -92415: 'BAMUM LETTER PHASE-D KEUAETMEUN', -92454: 'BAMUM LETTER PHASE-D KEUM', -92431: 'BAMUM LETTER PHASE-D KEUOT MBUAE', -92460: 'BAMUM LETTER PHASE-D KEUP', -92489: 'BAMUM LETTER PHASE-D KU', -92474: 'BAMUM LETTER PHASE-D KUN', -92422: 'BAMUM LETTER PHASE-D KUOM', -92479: 'BAMUM LETTER PHASE-D KUQ', -92449: 'BAMUM LETTER PHASE-D KWAET', -92502: 'BAMUM LETTER PHASE-D KYEE', -92495: 'BAMUM LETTER PHASE-D LEEEE', -92464: 'BAMUM LETTER PHASE-D LET', -92439: 'BAMUM LETTER PHASE-D LEUAEP', -92484: 'BAMUM LETTER PHASE-D LEUM', -92406: 'BAMUM LETTER PHASE-D LIEE', -92511: 'BAMUM LETTER PHASE-D LOQ', -92446: 'BAMUM LETTER PHASE-D LUM', -92497: 'BAMUM LETTER PHASE-D M', -92482: 'BAMUM LETTER PHASE-D MAENJET', -92426: 'BAMUM LETTER PHASE-D MALEERI', -92448: 'BAMUM LETTER PHASE-D MBAA', -92515: 'BAMUM LETTER PHASE-D MBAA SEVEN', -92401: 'BAMUM LETTER PHASE-D MBUO', -92496: 'BAMUM LETTER PHASE-D MEEEE', -92478: 'BAMUM LETTER PHASE-D MEUN', -92427: 'BAMUM LETTER PHASE-D MEUT', -92458: 'BAMUM LETTER PHASE-D MFEUAE', -92424: 'BAMUM LETTER PHASE-D MFEUT', -92466: 'BAMUM LETTER PHASE-D MFIEE', -92445: 'BAMUM LETTER PHASE-D MFO', -92404: 'BAMUM LETTER PHASE-D MFON', -92442: 'BAMUM LETTER PHASE-D MGBEUN', -92444: 'BAMUM LETTER PHASE-D MGBIEE', -92438: 'BAMUM LETTER PHASE-D MGBOFUM', -92441: 'BAMUM LETTER PHASE-D MONI', -92499: 'BAMUM LETTER PHASE-D MU', -92510: 'BAMUM LETTER PHASE-D MVOP', -92471: 'BAMUM LETTER PHASE-D NDAM', -92437: 'BAMUM LETTER PHASE-D NDEE', -92425: 'BAMUM LETTER PHASE-D NDEUX', -92440: 'BAMUM LETTER PHASE-D NDON', -92465: 'BAMUM LETTER PHASE-D NGGAAM', -92409: 'BAMUM LETTER PHASE-D NGGAAMAE', -92483: 'BAMUM LETTER PHASE-D NGGAP', -92475: 'BAMUM LETTER PHASE-D NGGEUX', -92485: 'BAMUM LETTER PHASE-D NGGUOM', -92467: 'BAMUM LETTER PHASE-D NGGWAEN', -92414: 'BAMUM LETTER PHASE-D NGKAP', -92457: 'BAMUM LETTER PHASE-D NGKEUAEQ', -92432: 'BAMUM LETTER PHASE-D NGKEURI', -92476: 'BAMUM LETTER PHASE-D NGKIEE', -92412: 'BAMUM LETTER PHASE-D NGKUN', -92435: 'BAMUM LETTER PHASE-D NGKYEE', -92507: 'BAMUM LETTER PHASE-D NI', -92418: 'BAMUM LETTER PHASE-D NJAP', -92430: 'BAMUM LETTER PHASE-D NJEUAEM', -92407: 'BAMUM LETTER PHASE-D NJEUT', -92403: 'BAMUM LETTER PHASE-D NJI', -92405: 'BAMUM LETTER PHASE-D NJIEE', -92487: 'BAMUM LETTER PHASE-D NJUEQ', -92408: 'BAMUM LETTER PHASE-D NSHEE', -92486: 'BAMUM LETTER PHASE-D NSHUT', -92447: 'BAMUM LETTER PHASE-D NSIEEP', -92459: 'BAMUM LETTER PHASE-D NSIEET', -92480: 'BAMUM LETTER PHASE-D NSUM', -92505: 'BAMUM LETTER PHASE-D NTEE', -92472: 'BAMUM LETTER PHASE-D NTEUM', -92514: 'BAMUM LETTER PHASE-D NTUU', -92503: 'BAMUM LETTER PHASE-D NU', -92410: 'BAMUM LETTER PHASE-D NYAM', -92450: 'BAMUM LETTER PHASE-D NYET', -92493: 'BAMUM LETTER PHASE-D NYI', -92463: 'BAMUM LETTER PHASE-D NYUE', -92469: 'BAMUM LETTER PHASE-D PAP', -92506: 'BAMUM LETTER PHASE-D PEE', -92462: 'BAMUM LETTER PHASE-D PEUTAE', -92461: 'BAMUM LETTER PHASE-D PIP', -92509: 'BAMUM LETTER PHASE-D PUQ', -92443: 'BAMUM LETTER PHASE-D PUUT', -92455: 'BAMUM LETTER PHASE-D RAEM', -92512: 'BAMUM LETTER PHASE-D REN MUCH', -92490: 'BAMUM LETTER PHASE-D REN OLD', -92494: 'BAMUM LETTER PHASE-D RII', -92423: 'BAMUM LETTER PHASE-D SAP', -92516: 'BAMUM LETTER PHASE-D SAQ', -92428: 'BAMUM LETTER PHASE-D SEUAEQ', -92413: 'BAMUM LETTER PHASE-D SHEE', -92417: 'BAMUM LETTER PHASE-D SHEUAE', -92501: 'BAMUM LETTER PHASE-D SHEUX', -92500: 'BAMUM LETTER PHASE-D SHII', -92508: 'BAMUM LETTER PHASE-D SHOQ', -92504: 'BAMUM LETTER PHASE-D SHU', -92452: 'BAMUM LETTER PHASE-D SOT', -92473: 'BAMUM LETTER PHASE-D SUAE', -92419: 'BAMUM LETTER PHASE-D SUE', -92498: 'BAMUM LETTER PHASE-D SUU', -92491: 'BAMUM LETTER PHASE-D TAE', -92456: 'BAMUM LETTER PHASE-D TEEEE', -92451: 'BAMUM LETTER PHASE-D TEUAEN', -92481: 'BAMUM LETTER PHASE-D TEUN', -92416: 'BAMUM LETTER PHASE-D TEUT', -92513: 'BAMUM LETTER PHASE-D TI', -92492: 'BAMUM LETTER PHASE-D TOQ', -92433: 'BAMUM LETTER PHASE-D TU', -92477: 'BAMUM LETTER PHASE-D TUOT', -92402: 'BAMUM LETTER PHASE-D WAP', -92411: 'BAMUM LETTER PHASE-D WUAEN', -92421: 'BAMUM LETTER PHASE-D YAEMMAE', -92429: 'BAMUM LETTER PHASE-D YEN', -92468: 'BAMUM LETTER PHASE-D YUOM', -92470: 'BAMUM LETTER PHASE-D YUOP', -92453: 'BAMUM LETTER PHASE-D YUWOQ', -92629: 'BAMUM LETTER PHASE-E A', -92603: 'BAMUM LETTER PHASE-E FA', -92673: 'BAMUM LETTER PHASE-E FAQ', -92651: 'BAMUM LETTER PHASE-E FEE', -92627: 'BAMUM LETTER PHASE-E FOM', -92626: 'BAMUM LETTER PHASE-E FU CALL', -92616: 'BAMUM LETTER PHASE-E FU I', -92661: 'BAMUM LETTER PHASE-E FU REMEDY', -92596: 'BAMUM LETTER PHASE-E FUE', -92612: 'BAMUM LETTER PHASE-E FUET', -92585: 'BAMUM LETTER PHASE-E GBET', -92597: 'BAMUM LETTER PHASE-E GBEUX', -92578: 'BAMUM LETTER PHASE-E GHAAMAE', -92602: 'BAMUM LETTER PHASE-E GHET', -92615: 'BAMUM LETTER PHASE-E GHEUAE', -92565: 'BAMUM LETTER PHASE-E GHEUN', -92552: 'BAMUM LETTER PHASE-E GHEUX', -92674: 'BAMUM LETTER PHASE-E GHOM', -92632: 'BAMUM LETTER PHASE-E I', -92599: 'BAMUM LETTER PHASE-E KET', -92570: 'BAMUM LETTER PHASE-E KEUAE', -92646: 'BAMUM LETTER PHASE-E KEUX', -92670: 'BAMUM LETTER PHASE-E KI', -92665: 'BAMUM LETTER PHASE-E KO', -92532: 'BAMUM LETTER PHASE-E KPEUX', -92587: 'BAMUM LETTER PHASE-E KUET', -92538: 'BAMUM LETTER PHASE-E KUOP', -92620: 'BAMUM LETTER PHASE-E KUT', -92575: 'BAMUM LETTER PHASE-E LAAM', -92521: 'BAMUM LETTER PHASE-E LAP', -92633: 'BAMUM LETTER PHASE-E LAQ', -92595: 'BAMUM LETTER PHASE-E LEUAEM', -92539: 'BAMUM LETTER PHASE-E LOM', -92523: 'BAMUM LETTER PHASE-E LOON', -92556: 'BAMUM LETTER PHASE-E LOOT', -92664: 'BAMUM LETTER PHASE-E LOQ', -92653: 'BAMUM LETTER PHASE-E LU', -92667: 'BAMUM LETTER PHASE-E MA', -92600: 'BAMUM LETTER PHASE-E MAE', -92542: 'BAMUM LETTER PHASE-E MAEM', -92555: 'BAMUM LETTER PHASE-E MAP', -92668: 'BAMUM LETTER PHASE-E MAQ', -92582: 'BAMUM LETTER PHASE-E MBEE', -92520: 'BAMUM LETTER PHASE-E MBEUM', -92666: 'BAMUM LETTER PHASE-E MEN', -92591: 'BAMUM LETTER PHASE-E MFEUQ', -92551: 'BAMUM LETTER PHASE-E MGBA', -92581: 'BAMUM LETTER PHASE-E MGBEN', -92654: 'BAMUM LETTER PHASE-E MI', -92611: 'BAMUM LETTER PHASE-E MIEE', -92671: 'BAMUM LETTER PHASE-E MON', -92614: 'BAMUM LETTER PHASE-E MUAE', -92617: 'BAMUM LETTER PHASE-E MVI', -92662: 'BAMUM LETTER PHASE-E NA', -92613: 'BAMUM LETTER PHASE-E NAE', -92637: 'BAMUM LETTER PHASE-E NDAA MY HOUSE', -92562: 'BAMUM LETTER PHASE-E NDAA SOFTNESS', -92518: 'BAMUM LETTER PHASE-E NDAP', -92592: 'BAMUM LETTER PHASE-E NDIAQ', -92558: 'BAMUM LETTER PHASE-E NDIQ', -92528: 'BAMUM LETTER PHASE-E NDUN', -92658: 'BAMUM LETTER PHASE-E NGA', -92579: 'BAMUM LETTER PHASE-E NGEUREUT', -92557: 'BAMUM LETTER PHASE-E NGGEEEE', -92607: 'BAMUM LETTER PHASE-E NGGEUAE', -92535: 'BAMUM LETTER PHASE-E NGGEUAET', -92563: 'BAMUM LETTER PHASE-E NGGUAESHAE NYAM', -92624: 'BAMUM LETTER PHASE-E NGGUP', -92550: 'BAMUM LETTER PHASE-E NGGURAE', -92531: 'BAMUM LETTER PHASE-E NGKA', -92601: 'BAMUM LETTER PHASE-E NGKAAMI', -92553: 'BAMUM LETTER PHASE-E NGKEUAEM', -92543: 'BAMUM LETTER PHASE-E NGKEUX', -92619: 'BAMUM LETTER PHASE-E NGKUM', -92598: 'BAMUM LETTER PHASE-E NGKUP', -92541: 'BAMUM LETTER PHASE-E NGOP', -92544: 'BAMUM LETTER PHASE-E NGOQ', -92640: 'BAMUM LETTER PHASE-E NGUAE', -92657: 'BAMUM LETTER PHASE-E NGUAET', -92554: 'BAMUM LETTER PHASE-E NJAEMLI', -92628: 'BAMUM LETTER PHASE-E NJEE', -92648: 'BAMUM LETTER PHASE-E NJEE EPOCH', -92547: 'BAMUM LETTER PHASE-E NJEUX', -92584: 'BAMUM LETTER PHASE-E NKOM', -92540: 'BAMUM LETTER PHASE-E NSHIEE', -92545: 'BAMUM LETTER PHASE-E NSHUE', -92527: 'BAMUM LETTER PHASE-E NSHUOP', -92622: 'BAMUM LETTER PHASE-E NTAP', -92604: 'BAMUM LETTER PHASE-E NTUM', -92608: 'BAMUM LETTER PHASE-E NYI BETWEEN', -92589: 'BAMUM LETTER PHASE-E NYI CLEAVER', -92583: 'BAMUM LETTER PHASE-E NZAQ', -92609: 'BAMUM LETTER PHASE-E NZUQ', -92631: 'BAMUM LETTER PHASE-E O', -92625: 'BAMUM LETTER PHASE-E PA PEOPLE', -92634: 'BAMUM LETTER PHASE-E PA PLURAL', -92524: 'BAMUM LETTER PHASE-E PAA', -92536: 'BAMUM LETTER PHASE-E PAAM', -92548: 'BAMUM LETTER PHASE-E PEEM', -92605: 'BAMUM LETTER PHASE-E PEUT', -92647: 'BAMUM LETTER PHASE-E PEUX', -92663: 'BAMUM LETTER PHASE-E PI', -92593: 'BAMUM LETTER PHASE-E PIEEQ', -92621: 'BAMUM LETTER PHASE-E PIET', -92568: 'BAMUM LETTER PHASE-E PO', -92610: 'BAMUM LETTER PHASE-E POON', -92576: 'BAMUM LETTER PHASE-E PU', -92529: 'BAMUM LETTER PHASE-E PUAE', -92618: 'BAMUM LETTER PHASE-E PUAQ', -92649: 'BAMUM LETTER PHASE-E PUE', -92561: 'BAMUM LETTER PHASE-E PUM', -92656: 'BAMUM LETTER PHASE-E RAE', -92526: 'BAMUM LETTER PHASE-E RAQ', -92655: 'BAMUM LETTER PHASE-E REUX', -92546: 'BAMUM LETTER PHASE-E RIMGBA', -92549: 'BAMUM LETTER PHASE-E SAA', -92534: 'BAMUM LETTER PHASE-E SEE', -92560: 'BAMUM LETTER PHASE-E SET', -92580: 'BAMUM LETTER PHASE-E SHEUAEQ', -92638: 'BAMUM LETTER PHASE-E SHIQ', -92659: 'BAMUM LETTER PHASE-E SHO', -92660: 'BAMUM LETTER PHASE-E SHOQ', -92525: 'BAMUM LETTER PHASE-E SOM', -92571: 'BAMUM LETTER PHASE-E SUAEN', -92635: 'BAMUM LETTER PHASE-E TAA', -92577: 'BAMUM LETTER PHASE-E TAAQ', -92559: 'BAMUM LETTER PHASE-E TAEN NTEUM', -92530: 'BAMUM LETTER PHASE-E TAM', -92636: 'BAMUM LETTER PHASE-E TAQ', -92672: 'BAMUM LETTER PHASE-E TEN', -92669: 'BAMUM LETTER PHASE-E TEU', -92572: 'BAMUM LETTER PHASE-E TEUAEQ', -92537: 'BAMUM LETTER PHASE-E TOO', -92519: 'BAMUM LETTER PHASE-E TOON', -92630: 'BAMUM LETTER PHASE-E TOQ', -92566: 'BAMUM LETTER PHASE-E TUAE', -92586: 'BAMUM LETTER PHASE-E TUM', -92569: 'BAMUM LETTER PHASE-E TUMAE', -92652: 'BAMUM LETTER PHASE-E VEE', -92573: 'BAMUM LETTER PHASE-E VEUAE', -92522: 'BAMUM LETTER PHASE-E VOM', -92574: 'BAMUM LETTER PHASE-E WEUX', -92650: 'BAMUM LETTER PHASE-E WUE', -92533: 'BAMUM LETTER PHASE-E WUO', -92588: 'BAMUM LETTER PHASE-E YAP', -92567: 'BAMUM LETTER PHASE-E YEUAE', -92623: 'BAMUM LETTER PHASE-E YEUAET', -92606: 'BAMUM LETTER PHASE-E YEUM', -92639: 'BAMUM LETTER PHASE-E YEUX', -92564: 'BAMUM LETTER PHASE-E YIEE', -92590: 'BAMUM LETTER PHASE-E YIT', -92643: 'BAMUM LETTER PHASE-E YOQ COVER', -92642: 'BAMUM LETTER PHASE-E YOQ SWIMMING', -92641: 'BAMUM LETTER PHASE-E YUAEN', -92594: 'BAMUM LETTER PHASE-E YUEQ', -92645: 'BAMUM LETTER PHASE-E YUN', -92644: 'BAMUM LETTER PHASE-E YUQ', -92678: 'BAMUM LETTER PHASE-F EE', -92715: 'BAMUM LETTER PHASE-F FOM', -92675: 'BAMUM LETTER PHASE-F KA', -92710: 'BAMUM LETTER PHASE-F KEN', -92695: 'BAMUM LETTER PHASE-F KET', -92719: 'BAMUM LETTER PHASE-F KO', -92726: 'BAMUM LETTER PHASE-F KPA', -92677: 'BAMUM LETTER PHASE-F KU', -92694: 'BAMUM LETTER PHASE-F KYEE', -92682: 'BAMUM LETTER PHASE-F LA', -92717: 'BAMUM LETTER PHASE-F LI', -92718: 'BAMUM LETTER PHASE-F LOQ', -92689: 'BAMUM LETTER PHASE-F M', -92722: 'BAMUM LETTER PHASE-F MA', -92724: 'BAMUM LETTER PHASE-F MBAA', -92720: 'BAMUM LETTER PHASE-F MBEN', -92685: 'BAMUM LETTER PHASE-F MEEEE', -92723: 'BAMUM LETTER PHASE-F MO', -92687: 'BAMUM LETTER PHASE-F NDAA', -92712: 'BAMUM LETTER PHASE-F NGGA', -92711: 'BAMUM LETTER PHASE-F NGKWAEN', -92708: 'BAMUM LETTER PHASE-F NI', -92688: 'BAMUM LETTER PHASE-F NJAEM', -92698: 'BAMUM LETTER PHASE-F NJUAE', -92702: 'BAMUM LETTER PHASE-F NSHA', -92704: 'BAMUM LETTER PHASE-F NTEE', -92697: 'BAMUM LETTER PHASE-F NU', -92696: 'BAMUM LETTER PHASE-F NUAE', -92681: 'BAMUM LETTER PHASE-F NYI', -92706: 'BAMUM LETTER PHASE-F PEE', -92703: 'BAMUM LETTER PHASE-F PEUX', -92714: 'BAMUM LETTER PHASE-F PUAE', -92679: 'BAMUM LETTER PHASE-F REE', -92721: 'BAMUM LETTER PHASE-F REN', -92709: 'BAMUM LETTER PHASE-F REUX', -92684: 'BAMUM LETTER PHASE-F RIEE', -92683: 'BAMUM LETTER PHASE-F RII', -92707: 'BAMUM LETTER PHASE-F RU', -92727: 'BAMUM LETTER PHASE-F SAMBA', -92693: 'BAMUM LETTER PHASE-F SEUX', -92691: 'BAMUM LETTER PHASE-F SHII', -92713: 'BAMUM LETTER PHASE-F SHO', -92700: 'BAMUM LETTER PHASE-F SHU', -92692: 'BAMUM LETTER PHASE-F SI', -92690: 'BAMUM LETTER PHASE-F SUU', -92686: 'BAMUM LETTER PHASE-F TAA', -92680: 'BAMUM LETTER PHASE-F TAE', -92725: 'BAMUM LETTER PHASE-F TET', -92676: 'BAMUM LETTER PHASE-F U', -92728: 'BAMUM LETTER PHASE-F VUEQ', -92716: 'BAMUM LETTER PHASE-F WA', -92705: 'BAMUM LETTER PHASE-F WUE', -92701: 'BAMUM LETTER PHASE-F YA', -92699: 'BAMUM LETTER PHASE-F YOQ', -127820: 'BANANA', -127974: 'BANK', -128181: 'BANKNOTE WITH DOLLAR SIGN', -128182: 'BANKNOTE WITH EURO SIGN', -128183: 'BANKNOTE WITH POUND SIGN', -128180: 'BANKNOTE WITH YEN SIGN', -128202: 'BAR CHART', -128136: 'BARBER POLE', -127936: 'BASKETBALL AND HOOP', -7153: 'BATAK CONSONANT SIGN H', -7152: 'BATAK CONSONANT SIGN NG', -7104: 'BATAK LETTER A', -7109: 'BATAK LETTER BA', -7137: 'BATAK LETTER CA', -7121: 'BATAK LETTER DA', -7118: 'BATAK LETTER GA', -7106: 'BATAK LETTER HA', -7140: 'BATAK LETTER I', -7120: 'BATAK LETTER JA', -7110: 'BATAK LETTER KARO BA', -7134: 'BATAK LETTER LA', -7124: 'BATAK LETTER MA', -7108: 'BATAK LETTER MANDAILING HA', -7114: 'BATAK LETTER MANDAILING NA', -7130: 'BATAK LETTER MANDAILING SA', -7139: 'BATAK LETTER MBA', -7113: 'BATAK LETTER NA', -7138: 'BATAK LETTER NDA', -7133: 'BATAK LETTER NGA', -7127: 'BATAK LETTER NORTHERN TA', -7136: 'BATAK LETTER NYA', -7111: 'BATAK LETTER PA', -7117: 'BATAK LETTER PAKPAK WA', -7122: 'BATAK LETTER RA', -7128: 'BATAK LETTER SA', -7105: 'BATAK LETTER SIMALUNGUN A', -7119: 'BATAK LETTER SIMALUNGUN GA', -7107: 'BATAK LETTER SIMALUNGUN HA', -7135: 'BATAK LETTER SIMALUNGUN LA', -7125: 'BATAK LETTER SIMALUNGUN MA', -7112: 'BATAK LETTER SIMALUNGUN PA', -7123: 'BATAK LETTER SIMALUNGUN RA', -7129: 'BATAK LETTER SIMALUNGUN SA', -7116: 'BATAK LETTER SIMALUNGUN WA', -7132: 'BATAK LETTER SIMALUNGUN YA', -7126: 'BATAK LETTER SOUTHERN TA', -7141: 'BATAK LETTER U', -7115: 'BATAK LETTER WA', -7131: 'BATAK LETTER YA', -7154: 'BATAK PANGOLAT', -7155: 'BATAK PANONGONAN', -7142: 'BATAK SIGN TOMPI', -7166: 'BATAK SYMBOL BINDU JUDUL', -7164: 'BATAK SYMBOL BINDU NA METEK', -7167: 'BATAK SYMBOL BINDU PANGOLAT', -7165: 'BATAK SYMBOL BINDU PINARBORAS', -7143: 'BATAK VOWEL SIGN E', -7145: 'BATAK VOWEL SIGN EE', -7146: 'BATAK VOWEL SIGN I', -7147: 'BATAK VOWEL SIGN KARO I', -7149: 'BATAK VOWEL SIGN KARO O', -7148: 'BATAK VOWEL SIGN O', -7144: 'BATAK VOWEL SIGN PAKPAK E', -7150: 'BATAK VOWEL SIGN U', -7151: 'BATAK VOWEL SIGN U FOR SIMALUNGUN SA', -128704: 'BATH', -128705: 'BATHTUB', -128267: 'BATTERY', -128059: 'BEAR FACE', -128147: 'BEATING HEART', -127866: 'BEER MUG', -128276: 'BELL', -128277: 'BELL WITH CANCELLATION STROKE', -983621: 'BENGALI LETTER KHINYA', -127857: 'BENTO BOX', -128690: 'BICYCLE', -128692: 'BICYCLIST', -128089: 'BIKINI', -127921: 'BILLIARDS', -128038: 'BIRD', -127874: 'BIRTHDAY CAKE', -9196: 'BLACK DOWN-POINTING DOUBLE TRIANGLE', -9194: 'BLACK LEFT-POINTING DOUBLE TRIANGLE', -9198: 'BLACK LEFT-POINTING DOUBLE TRIANGLE WITH VERTICAL BAR', -10067: 'BLACK QUESTION MARK ORNAMENT', -9193: 'BLACK RIGHT-POINTING DOUBLE TRIANGLE', -9197: 'BLACK RIGHT-POINTING DOUBLE TRIANGLE WITH VERTICAL BAR', -9199: 'BLACK RIGHT-POINTING TRIANGLE WITH DOUBLE VERTICAL BAR', -128306: 'BLACK SQUARE BUTTON', -9195: 'BLACK UP-POINTING DOUBLE TRIANGLE', -127804: 'BLOSSOM', -128033: 'BLOWFISH', -128216: 'BLUE BOOK', -128153: 'BLUE HEART', -128023: 'BOAR', -128163: 'BOMB', -128278: 'BOOKMARK', -128209: 'BOOKMARK TABS', -128218: 'BOOKS', -12728: 'BOPOMOFO LETTER GH', -12729: 'BOPOMOFO LETTER LH', -12730: 'BOPOMOFO LETTER ZY', -128144: 'BOUQUET', -127923: 'BOWLING', -128102: 'BOY', -69703: 'BRAHMI DANDA', -69742: 'BRAHMI DIGIT EIGHT', -69739: 'BRAHMI DIGIT FIVE', -69738: 'BRAHMI DIGIT FOUR', -69743: 'BRAHMI DIGIT NINE', -69735: 'BRAHMI DIGIT ONE', -69741: 'BRAHMI DIGIT SEVEN', -69740: 'BRAHMI DIGIT SIX', -69737: 'BRAHMI DIGIT THREE', -69736: 'BRAHMI DIGIT TWO', -69734: 'BRAHMI DIGIT ZERO', -69704: 'BRAHMI DOUBLE DANDA', -69637: 'BRAHMI LETTER A', -69638: 'BRAHMI LETTER AA', -69648: 'BRAHMI LETTER AI', -69650: 'BRAHMI LETTER AU', -69673: 'BRAHMI LETTER BA', -69674: 'BRAHMI LETTER BHA', -69656: 'BRAHMI LETTER CA', -69657: 'BRAHMI LETTER CHA', -69668: 'BRAHMI LETTER DA', -69663: 'BRAHMI LETTER DDA', -69664: 'BRAHMI LETTER DDHA', -69669: 'BRAHMI LETTER DHA', -69647: 'BRAHMI LETTER E', -69653: 'BRAHMI LETTER GA', -69654: 'BRAHMI LETTER GHA', -69683: 'BRAHMI LETTER HA', -69639: 'BRAHMI LETTER I', -69640: 'BRAHMI LETTER II', -69658: 'BRAHMI LETTER JA', -69659: 'BRAHMI LETTER JHA', -69651: 'BRAHMI LETTER KA', -69652: 'BRAHMI LETTER KHA', -69678: 'BRAHMI LETTER LA', -69684: 'BRAHMI LETTER LLA', -69675: 'BRAHMI LETTER MA', -69670: 'BRAHMI LETTER NA', -69655: 'BRAHMI LETTER NGA', -69665: 'BRAHMI LETTER NNA', -69660: 'BRAHMI LETTER NYA', -69649: 'BRAHMI LETTER O', -69685: 'BRAHMI LETTER OLD TAMIL LLLA', -69687: 'BRAHMI LETTER OLD TAMIL NNNA', -69686: 'BRAHMI LETTER OLD TAMIL RRA', -69671: 'BRAHMI LETTER PA', -69672: 'BRAHMI LETTER PHA', -69677: 'BRAHMI LETTER RA', -69682: 'BRAHMI LETTER SA', -69680: 'BRAHMI LETTER SHA', -69681: 'BRAHMI LETTER SSA', -69666: 'BRAHMI LETTER TA', -69667: 'BRAHMI LETTER THA', -69661: 'BRAHMI LETTER TTA', -69662: 'BRAHMI LETTER TTHA', -69641: 'BRAHMI LETTER U', -69642: 'BRAHMI LETTER UU', -69679: 'BRAHMI LETTER VA', -69645: 'BRAHMI LETTER VOCALIC L', -69646: 'BRAHMI LETTER VOCALIC LL', -69643: 'BRAHMI LETTER VOCALIC R', -69644: 'BRAHMI LETTER VOCALIC RR', -69676: 'BRAHMI LETTER YA', -69721: 'BRAHMI NUMBER EIGHT', -69730: 'BRAHMI NUMBER EIGHTY', -69727: 'BRAHMI NUMBER FIFTY', -69718: 'BRAHMI NUMBER FIVE', -69726: 'BRAHMI NUMBER FORTY', -69717: 'BRAHMI NUMBER FOUR', -69722: 'BRAHMI NUMBER NINE', -69731: 'BRAHMI NUMBER NINETY', -69714: 'BRAHMI NUMBER ONE', -69732: 'BRAHMI NUMBER ONE HUNDRED', -69733: 'BRAHMI NUMBER ONE THOUSAND', -69720: 'BRAHMI NUMBER SEVEN', -69729: 'BRAHMI NUMBER SEVENTY', -69719: 'BRAHMI NUMBER SIX', -69728: 'BRAHMI NUMBER SIXTY', -69723: 'BRAHMI NUMBER TEN', -69725: 'BRAHMI NUMBER THIRTY', -69716: 'BRAHMI NUMBER THREE', -69724: 'BRAHMI NUMBER TWENTY', -69715: 'BRAHMI NUMBER TWO', -69708: 'BRAHMI PUNCTUATION CRESCENT BAR', -69705: 'BRAHMI PUNCTUATION DOT', -69706: 'BRAHMI PUNCTUATION DOUBLE DOT', -69707: 'BRAHMI PUNCTUATION LINE', -69709: 'BRAHMI PUNCTUATION LOTUS', -69633: 'BRAHMI SIGN ANUSVARA', -69632: 'BRAHMI SIGN CANDRABINDU', -69635: 'BRAHMI SIGN JIHVAMULIYA', -69636: 'BRAHMI SIGN UPADHMANIYA', -69634: 'BRAHMI SIGN VISARGA', -69702: 'BRAHMI VIRAMA', -69688: 'BRAHMI VOWEL SIGN AA', -69699: 'BRAHMI VOWEL SIGN AI', -69701: 'BRAHMI VOWEL SIGN AU', -69689: 'BRAHMI VOWEL SIGN BHATTIPROLU AA', -69698: 'BRAHMI VOWEL SIGN E', -69690: 'BRAHMI VOWEL SIGN I', -69691: 'BRAHMI VOWEL SIGN II', -69700: 'BRAHMI VOWEL SIGN O', -69692: 'BRAHMI VOWEL SIGN U', -69693: 'BRAHMI VOWEL SIGN UU', -69696: 'BRAHMI VOWEL SIGN VOCALIC L', -69697: 'BRAHMI VOWEL SIGN VOCALIC LL', -69694: 'BRAHMI VOWEL SIGN VOCALIC R', -69695: 'BRAHMI VOWEL SIGN VOCALIC RR', -127838: 'BREAD', -128112: 'BRIDE WITH VEIL', -127753: 'BRIDGE AT NIGHT', -128188: 'BRIEFCASE', -128148: 'BROKEN HEART', -128027: 'BUG', -128652: 'BUS', -128655: 'BUS STOP', -128100: 'BUST IN SILHOUETTE', -128101: 'BUSTS IN SILHOUETTE', -127797: 'CACTUS', -128197: 'CALENDAR', -128247: 'CAMERA', -127852: 'CANDY', -128199: 'CARD INDEX', -127904: 'CAROUSEL HORSE', -127887: 'CARP STREAMER', -128008: 'CAT', -128049: 'CAT FACE', -128569: 'CAT FACE WITH TEARS OF JOY', -128572: 'CAT FACE WITH WRY SMILE', -128201: 'CHART WITH DOWNWARDS TREND', -128200: 'CHART WITH UPWARDS TREND', -128185: 'CHART WITH UPWARDS TREND AND YEN SIGN', -128227: 'CHEERING MEGAPHONE', -127937: 'CHEQUERED FLAG', -127826: 'CHERRIES', -127800: 'CHERRY BLOSSOM', -127792: 'CHESTNUT', -128020: 'CHICKEN', -128696: 'CHILDREN CROSSING', -127851: 'CHOCOLATE BAR', -127876: 'CHRISTMAS TREE', -127910: 'CINEMA', -127569: 'CIRCLED IDEOGRAPH ACCEPT', -127568: 'CIRCLED IDEOGRAPH ADVANTAGE', -127914: 'CIRCUS TENT', -127750: 'CITYSCAPE AT DUSK', -127916: 'CLAPPER BOARD', -128079: 'CLAPPING HANDS SIGN', -127867: 'CLINKING BEER MUGS', -128203: 'CLIPBOARD', -128343: 'CLOCK FACE EIGHT OCLOCK', -128355: 'CLOCK FACE EIGHT-THIRTY', -128346: 'CLOCK FACE ELEVEN OCLOCK', -128358: 'CLOCK FACE ELEVEN-THIRTY', -128340: 'CLOCK FACE FIVE OCLOCK', -128352: 'CLOCK FACE FIVE-THIRTY', -128339: 'CLOCK FACE FOUR OCLOCK', -128351: 'CLOCK FACE FOUR-THIRTY', -128344: 'CLOCK FACE NINE OCLOCK', -128356: 'CLOCK FACE NINE-THIRTY', -128336: 'CLOCK FACE ONE OCLOCK', -128348: 'CLOCK FACE ONE-THIRTY', -128342: 'CLOCK FACE SEVEN OCLOCK', -128354: 'CLOCK FACE SEVEN-THIRTY', -128341: 'CLOCK FACE SIX OCLOCK', -128353: 'CLOCK FACE SIX-THIRTY', -128345: 'CLOCK FACE TEN OCLOCK', -128357: 'CLOCK FACE TEN-THIRTY', -128338: 'CLOCK FACE THREE OCLOCK', -128350: 'CLOCK FACE THREE-THIRTY', -128347: 'CLOCK FACE TWELVE OCLOCK', -128359: 'CLOCK FACE TWELVE-THIRTY', -128337: 'CLOCK FACE TWO OCLOCK', -128349: 'CLOCK FACE TWO-THIRTY', -128259: 'CLOCKWISE DOWNWARDS AND UPWARDS OPEN CIRCLE ARROWS', -128257: 'CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE ARROWS', -128258: 'CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE ARROWS WITH CIRCLED ONE OVERLAY', -128213: 'CLOSED BOOK', -128272: 'CLOSED LOCK WITH KEY', -128234: 'CLOSED MAILBOX WITH LOWERED FLAG', -128235: 'CLOSED MAILBOX WITH RAISED FLAG', -127746: 'CLOSED UMBRELLA', -127864: 'COCKTAIL GLASS', -128165: 'COLLISION SYMBOL', -7676: 'COMBINING DOUBLE INVERTED BREVE BELOW', -127882: 'CONFETTI BALL', -128534: 'CONFOUNDED FACE', -128679: 'CONSTRUCTION SIGN', -128119: 'CONSTRUCTION WORKER', -127978: 'CONVENIENCE STORE', -127834: 'COOKED RICE', -127850: 'COOKIE', -127859: 'COOKING', -128145: 'COUPLE WITH HEART', -128004: 'COW', -128046: 'COW FACE', -128179: 'CREDIT CARD', -127769: 'CRESCENT MOON', -128010: 'CROCODILE', -10060: 'CROSS MARK', -127884: 'CROSSED FLAGS', -128081: 'CROWN', -128575: 'CRYING CAT FACE', -128546: 'CRYING FACE', -128302: 'CRYSTAL BALL', -10160: 'CURLY LOOP', -128177: 'CURRENCY EXCHANGE', -127835: 'CURRY AND RICE', -127854: 'CUSTARD', -128707: 'CUSTOMS', -127744: 'CYCLONE', -42592: 'CYRILLIC CAPITAL LETTER REVERSED TSE', -1318: 'CYRILLIC CAPITAL LETTER SHHA WITH DESCENDER', -42593: 'CYRILLIC SMALL LETTER REVERSED TSE', -1319: 'CYRILLIC SMALL LETTER SHHA WITH DESCENDER', -128131: 'DANCER', -127841: 'DANGO', -128168: 'DASH SYMBOL', -127795: 'DECIDUOUS TREE', -128666: 'DELIVERY TRUCK', -127980: 'DEPARTMENT STORE', -2421: 'DEVANAGARI LETTER AW', -2419: 'DEVANAGARI LETTER OE', -2420: 'DEVANAGARI LETTER OOE', -2422: 'DEVANAGARI LETTER UE', -2423: 'DEVANAGARI LETTER UUE', -2383: 'DEVANAGARI VOWEL SIGN AW', -2362: 'DEVANAGARI VOWEL SIGN OE', -2363: 'DEVANAGARI VOWEL SIGN OOE', -2390: 'DEVANAGARI VOWEL SIGN UE', -2391: 'DEVANAGARI VOWEL SIGN UUE', -128160: 'DIAMOND SHAPE WITH A DOT INSIDE', -127919: 'DIRECT HIT', -128549: 'DISAPPOINTED BUT RELIEVED FACE', -128542: 'DISAPPOINTED FACE', -128565: 'DIZZY FACE', -128171: 'DIZZY SYMBOL', -128687: 'DO NOT LITTER SYMBOL', -128021: 'DOG', -128054: 'DOG FACE', -128044: 'DOLPHIN', -128682: 'DOOR', -10175: 'DOUBLE CURLY LOOP', -127849: 'DOUGHNUT', -128315: 'DOWN-POINTING RED TRIANGLE', -128317: 'DOWN-POINTING SMALL RED TRIANGLE', -128009: 'DRAGON', -128050: 'DRAGON FACE', -128087: 'DRESS', -128042: 'DROMEDARY CAMEL', -128167: 'DROPLET', -128192: 'DVD', -128231: 'E-MAIL SYMBOL', -128066: 'EAR', -127805: 'EAR OF MAIZE', -127806: 'EAR OF RICE', -127758: 'EARTH GLOBE AMERICAS', -127759: 'EARTH GLOBE ASIA-AUSTRALIA', -127757: 'EARTH GLOBE EUROPE-AFRICA', -128161: 'ELECTRIC LIGHT BULB', -128268: 'ELECTRIC PLUG', -128294: 'ELECTRIC TORCH', -128024: 'ELEPHANT', -128282: 'END WITH LEFTWARDS ARROW ABOVE', -128233: 'ENVELOPE WITH DOWNWARDS ARROW ABOVE', -4957: 'ETHIOPIC COMBINING GEMINATION AND VOWEL LENGTH MARK', -4958: 'ETHIOPIC COMBINING VOWEL LENGTH MARK', -43816: 'ETHIOPIC SYLLABLE BBA', -43819: 'ETHIOPIC SYLLABLE BBAA', -43821: 'ETHIOPIC SYLLABLE BBE', -43820: 'ETHIOPIC SYLLABLE BBEE', -43818: 'ETHIOPIC SYLLABLE BBI', -43822: 'ETHIOPIC SYLLABLE BBO', -43817: 'ETHIOPIC SYLLABLE BBU', -43808: 'ETHIOPIC SYLLABLE CCHHA', -43811: 'ETHIOPIC SYLLABLE CCHHAA', -43813: 'ETHIOPIC SYLLABLE CCHHE', -43812: 'ETHIOPIC SYLLABLE CCHHEE', -43810: 'ETHIOPIC SYLLABLE CCHHI', -43814: 'ETHIOPIC SYLLABLE CCHHO', -43809: 'ETHIOPIC SYLLABLE CCHHU', -43787: 'ETHIOPIC SYLLABLE DDHAA', -43789: 'ETHIOPIC SYLLABLE DDHE', -43788: 'ETHIOPIC SYLLABLE DDHEE', -43786: 'ETHIOPIC SYLLABLE DDHI', -43790: 'ETHIOPIC SYLLABLE DDHO', -43785: 'ETHIOPIC SYLLABLE DDHU', -43795: 'ETHIOPIC SYLLABLE DZAA', -43797: 'ETHIOPIC SYLLABLE DZE', -43796: 'ETHIOPIC SYLLABLE DZEE', -43794: 'ETHIOPIC SYLLABLE DZI', -43798: 'ETHIOPIC SYLLABLE DZO', -43793: 'ETHIOPIC SYLLABLE DZU', -43779: 'ETHIOPIC SYLLABLE TTHAA', -43781: 'ETHIOPIC SYLLABLE TTHE', -43780: 'ETHIOPIC SYLLABLE TTHEE', -43778: 'ETHIOPIC SYLLABLE TTHI', -43782: 'ETHIOPIC SYLLABLE TTHO', -43777: 'ETHIOPIC SYLLABLE TTHU', -127984: 'EUROPEAN CASTLE', -127972: 'EUROPEAN POST OFFICE', -127794: 'EVERGREEN TREE', -128125: 'EXTRATERRESTRIAL ALIEN', -128083: 'EYEGLASSES', -128064: 'EYES', -128134: 'FACE MASSAGE', -128523: 'FACE SAVOURING DELICIOUS FOOD', -128561: 'FACE SCREAMING IN FEAR', -128536: 'FACE THROWING A KISS', -128531: 'FACE WITH COLD SWEAT', -128548: 'FACE WITH LOOK OF TRIUMPH', -128567: 'FACE WITH MEDICAL MASK', -128581: 'FACE WITH NO GOOD GESTURE', -128582: 'FACE WITH OK GESTURE', -128560: 'FACE WITH OPEN MOUTH AND COLD SWEAT', -128541: 'FACE WITH STUCK-OUT TONGUE AND TIGHTLY-CLOSED EYES', -128540: 'FACE WITH STUCK-OUT TONGUE AND WINKING EYE', -128514: 'FACE WITH TEARS OF JOY', -128566: 'FACE WITHOUT MOUTH', -127981: 'FACTORY', -127810: 'FALLEN LEAF', -128106: 'FAMILY', -127877: 'FATHER CHRISTMAS', -128224: 'FAX MACHINE', -128552: 'FEARFUL FACE', -127905: 'FERRIS WHEEL', -128193: 'FILE FOLDER', -128293: 'FIRE', -128658: 'FIRE ENGINE', -127879: 'FIREWORK SPARKLER', -127878: 'FIREWORKS', -127763: 'FIRST QUARTER MOON SYMBOL', -127771: 'FIRST QUARTER MOON WITH FACE', -128031: 'FISH', -127845: 'FISH CAKE WITH SWIRL DESIGN', -127907: 'FISHING POLE AND FISH', -128074: 'FISTED HAND SIGN', -128170: 'FLEXED BICEPS', -128190: 'FLOPPY DISK', -127924: 'FLOWER PLAYING CARDS', -128563: 'FLUSHED FACE', -127745: 'FOGGY', -128099: 'FOOTPRINTS', -127860: 'FORK AND KNIFE', -127808: 'FOUR LEAF CLOVER', -127839: 'FRENCH FRIES', -127844: 'FRIED SHRIMP', -128056: 'FROG FACE', From pypy.commits at gmail.com Fri Mar 1 08:14:13 2019 From: pypy.commits at gmail.com (stevie_92) Date: Fri, 01 Mar 2019 05:14:13 -0800 (PST) Subject: [pypy-commit] pypy cpyext-gc-cycle: Implemented tests for gc.garbage list and adapted interface Message-ID: <5c793025.1c69fb81.4d36d.bc1e@mx.google.com> Author: Stefan Beyer Branch: cpyext-gc-cycle Changeset: r96197:15bed0a0192d Date: 2019-03-01 14:13 +0100 http://bitbucket.org/pypy/pypy/changeset/15bed0a0192d/ Log: Implemented tests for gc.garbage list and adapted interface diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py --- a/pypy/module/cpyext/state.py +++ b/pypy/module/cpyext/state.py @@ -77,6 +77,7 @@ space = self.space if not self.space.config.translating: def dealloc_trigger(): + from pypy.interpreter.baseobjspace import W_Root from pypy.module.cpyext.pyobject import PyObject, decref, \ incref, cts, finalize, from_ref w_list = space.getattr(space.builtin_modules['gc'], @@ -116,10 +117,15 @@ llmemory.cast_ptr_to_adr(head)): rawrefcount.cyclic_garbage_remove() while True: - py_obj = rawrefcount.next_garbage(PyObject) + w_obj = rawrefcount.next_garbage_pypy(W_Root) if not py_obj: break - w_obj = from_ref(space, py_obj) + w_list.append(w_obj) + while True: + w_pyobj = rawrefcount.next_garbage_pyobj(PyObject) + if not py_obj: + break + w_obj = from_ref(space, w_pyobj) w_list.append(w_obj) print 'dealloc_trigger DONE' return "RETRY" @@ -282,6 +288,7 @@ def _rawrefcount_perform(space): + from pypy.interpreter.baseobjspace import W_Root from pypy.module.cpyext.pyobject import (PyObject, incref, decref, finalize, from_ref) @@ -315,10 +322,15 @@ rawrefcount.cyclic_garbage_remove() while True: - py_obj = rawrefcount.next_garbage(PyObject) + w_obj = rawrefcount.next_garbage_pypy(W_Root) if not py_obj: break - w_obj = from_ref(space, py_obj) + w_list.append(w_obj) + while True: + w_pyobj = rawrefcount.next_garbage_pyobj(PyObject) + if not py_obj: + break + w_obj = from_ref(space, w_pyobj) w_list.append(w_obj) class PyObjDeallocAction(executioncontext.AsyncAction): diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -3217,7 +3217,22 @@ gchdr.c_gc_next = next next.c_gc_prev = gchdr - def rawrefcount_next_garbage(self): + def rawrefcount_next_garbage_pypy(self): + # return the next pypy object which is only reachable from garbage + # pyobjects, probably need two more colors for this. one for marking + # so that they stay alive during sweep, one for marking, so they do not + # get returned here again + return lltype.nullptr(llmemory.GCREF.TO) + + def rawrefcount_next_garbage_pyobj(self): + # implement st objects in this list still remain in the set of + # all pyobjs, because references could still change and cause them + # to live again. also keep in mind, that state will create references + # to pyobjs in this list and might increment the refcount. + + # use create_link_pyobj on the result to create gc objects for pyobjects + #p = W_Root(42) + #p.pyobj = ob return llmemory.NULL diff --git a/rpython/memory/gc/test/dot/garbage_cpython_simple_1.dot b/rpython/memory/gc/test/dot/garbage_cpython_simple_1.dot new file mode 100644 --- /dev/null +++ b/rpython/memory/gc/test/dot/garbage_cpython_simple_1.dot @@ -0,0 +1,6 @@ +digraph G { + "a" [type=C, alive=y, garbage=y, finalizer=legacy]; + "b" [type=C, alive=y, garbage=y]; + "a" -> "b"; + "b" -> "a"; +} \ No newline at end of file diff --git a/rpython/memory/gc/test/dot/garbage_cross_simple_1.dot b/rpython/memory/gc/test/dot/garbage_cross_simple_1.dot new file mode 100644 --- /dev/null +++ b/rpython/memory/gc/test/dot/garbage_cross_simple_1.dot @@ -0,0 +1,10 @@ +digraph G { + "a" [type=C, alive=y, garbage=y, finalizer=legacy]; + "b" [type=B, alive=y, garbage=y]; + "c" [type=P, alive=y, garbage=y]; + "d" [type=P, alive=y, rooted=y]; + "a" -> "b"; + "b" -> "a"; + "b" -> "c"; + "c" -> "d"; +} \ No newline at end of file diff --git a/rpython/memory/gc/test/test_rawrefcount.py b/rpython/memory/gc/test/test_rawrefcount.py --- a/rpython/memory/gc/test/test_rawrefcount.py +++ b/rpython/memory/gc/test/test_rawrefcount.py @@ -1,17 +1,17 @@ import os, py from rpython.rtyper.lltypesystem import lltype, llmemory, rffi -from rpython.memory.gc.incminimark import IncrementalMiniMarkGC +from rpython.memory.gc.incminimark import IncrementalMiniMarkGC as IncMiniMark from rpython.memory.gc.test.test_direct import BaseDirectGCTest -from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY -from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT -PYOBJ_HDR = IncrementalMiniMarkGC.PYOBJ_HDR -PYOBJ_HDR_PTR = IncrementalMiniMarkGC.PYOBJ_HDR_PTR -RAWREFCOUNT_VISIT = IncrementalMiniMarkGC.RAWREFCOUNT_VISIT -PYOBJ_GC_HDR = IncrementalMiniMarkGC.PYOBJ_GC_HDR -PYOBJ_GC_HDR_PTR = IncrementalMiniMarkGC.PYOBJ_GC_HDR_PTR -RAWREFCOUNT_FINALIZER_MODERN = \ - IncrementalMiniMarkGC.RAWREFCOUNT_FINALIZER_MODERN -RAWREFCOUNT_FINALIZER_NONE = IncrementalMiniMarkGC.RAWREFCOUNT_FINALIZER_NONE +from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY, REFCNT_FROM_PYPY_LIGHT + +PYOBJ_HDR = IncMiniMark.PYOBJ_HDR +PYOBJ_HDR_PTR = IncMiniMark.PYOBJ_HDR_PTR +RAWREFCOUNT_VISIT = IncMiniMark.RAWREFCOUNT_VISIT +PYOBJ_GC_HDR = IncMiniMark.PYOBJ_GC_HDR +PYOBJ_GC_HDR_PTR = IncMiniMark.PYOBJ_GC_HDR_PTR +RAWREFCOUNT_FINALIZER_MODERN = IncMiniMark.RAWREFCOUNT_FINALIZER_MODERN +RAWREFCOUNT_FINALIZER_LEGACY = IncMiniMark.RAWREFCOUNT_FINALIZER_LEGACY +RAWREFCOUNT_FINALIZER_NONE = IncMiniMark.RAWREFCOUNT_FINALIZER_NONE S = lltype.GcForwardReference() S.become(lltype.GcStruct('S', @@ -21,7 +21,7 @@ class TestRawRefCount(BaseDirectGCTest): - GCClass = IncrementalMiniMarkGC + GCClass = IncMiniMark def setup_method(self, method): BaseDirectGCTest.setup_method(self, method) @@ -448,13 +448,14 @@ class NodeInfo: def __init__(self, type, alive, ext_refcnt, finalizer, resurrect, - delete): + delete, garbage): self.type = type self.alive = alive self.ext_refcnt = ext_refcnt self.finalizer = finalizer self.resurrect = resurrect self.delete = delete + self.garbage = garbage path = os.path.join(self.dot_dir, file) g = pydot.graph_from_dot_file(path)[0] @@ -475,8 +476,9 @@ finalizers = True resurrect = attr['resurrect'] if 'resurrect' in attr else None delete = attr['delete'] if 'delete' in attr else None + garbage = True if 'garbage' in attr else False info = NodeInfo(type, alive, ext_refcnt, finalizer, resurrect, - delete) + delete, garbage) if type == "C": r, raddr, check_alive = self._rawrefcount_pyobj() r.c_ob_refcnt += ext_refcnt @@ -526,6 +528,8 @@ nodes[resurrect].info.ext_refcnt += 1 if delete is not None: self._rawrefcount_add_delete(n.r, nodes[delete].r) + elif n.info.finalizer == "legacy": + self.pyobj_finalizer[index] = RAWREFCOUNT_FINALIZER_LEGACY else: self.pyobj_finalizer[index] = RAWREFCOUNT_FINALIZER_NONE @@ -545,6 +549,8 @@ self.gc.rrc_tp_traverse(source.r, append, None) assert len(dests_target) == 0 + garbage_pypy = [] + garbage_pyobj = [] def cleanup(): # do cleanup after collection (clear all dead pyobjects) def finalize_modern(pyobj): @@ -621,6 +627,13 @@ self.gc.rawrefcount_cyclic_garbage_remove() next_dead = self.gc.rawrefcount_cyclic_garbage_head() + next = self.gc.rawrefcount_next_garbage_pypy() + while next <> lltype.nullptr(llmemory.GCREF.TO): + garbage_pypy.append(next) + next = self.gc.rawrefcount_next_garbage_pyobj() + while next <> llmemory.NULL: + garbage_pyobj.append(next) + # do a collection to find cyclic isolates and clean them, if there are # no finalizers self.gc.collect() @@ -649,3 +662,16 @@ py.test.raises(RuntimeError, "n.p.x") # dead else: py.test.raises(RuntimeError, "n.r.c_ob_refcnt") # dead + + # check if unreachable objects in cyclic structures with legacy + # finalizers and all otherwise unreachable objects reachable from them + # have been added to the garbage list + for name in nodes: + n = nodes[name] + if n.info.alive: + if n.info.type == "C": + assert n.info.garbage != (n.raddr not in garbage_pyobj) + else: + assert n.info.garbage != (n.pref not in garbage_pypy) + else: + assert not n.info.garbage \ No newline at end of file diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -518,8 +518,11 @@ self.rawrefcount_cyclic_garbage_remove_ptr = getfn( GCClass.rawrefcount_cyclic_garbage_remove, [s_gc], annmodel.s_None, inline = True) - self.rawrefcount_next_garbage_ptr = getfn( - GCClass.rawrefcount_next_garbage, [s_gc], + self.rawrefcount_next_garbage_pypy_ptr = getfn( + GCClass.rawrefcount_next_garbage_pypy, [s_gc], + s_gcref, inline = True) + self.rawrefcount_next_garbage_pyobj_ptr = getfn( + GCClass.rawrefcount_next_garbage_pyobj, [s_gc], SomeAddress(), inline = True) if GCClass.can_usually_pin_objects: @@ -1429,10 +1432,16 @@ [self.rawrefcount_cyclic_garbage_remove_ptr, self.c_const_gc]) - def gct_gc_rawrefcount_next_garbage(self, hop): + def gct_gc_rawrefcount_next_garbage_pypy(self, hop): + assert hop.spaceop.result.concretetype == llmemory.GCREF + hop.genop("direct_call", + [self.rawrefcount_next_garbage_pypy_ptr, self.c_const_gc], + resultvar=hop.spaceop.result) + + def gct_gc_rawrefcount_next_garbage_pyobj(self, hop): assert hop.spaceop.result.concretetype == llmemory.Address hop.genop("direct_call", - [self.rawrefcount_next_garbage_ptr, self.c_const_gc], + [self.rawrefcount_next_garbage_pyobj_ptr, self.c_const_gc], resultvar=hop.spaceop.result) def _set_into_gc_array_part(self, op): diff --git a/rpython/rlib/rawrefcount.py b/rpython/rlib/rawrefcount.py --- a/rpython/rlib/rawrefcount.py +++ b/rpython/rlib/rawrefcount.py @@ -146,7 +146,11 @@ return lltype.nullptr(OB_PTR_TYPE.TO) @not_rpython -def next_garbage(OB_PTR_TYPE): +def next_garbage_pypy(Class): + return lltype.nullptr(llmemory.GCREF.TO) + + at not_rpython +def next_garbage_pyobj(OB_PTR_TYPE): return lltype.nullptr(OB_PTR_TYPE.TO) @not_rpython @@ -372,7 +376,7 @@ class Entry(ExtRegistryEntry): _about_ = (next_dead, cyclic_garbage_head, next_cyclic_isolate, - next_garbage) + next_garbage_pyobj) def compute_result_annotation(self, s_OB_PTR_TYPE): from rpython.rtyper.llannotation import lltype_to_annotation @@ -386,13 +390,28 @@ name = 'gc_rawrefcount_cyclic_garbage_head' elif self.instance is next_cyclic_isolate: name = 'gc_rawrefcount_next_cyclic_isolate' - elif self.instance is next_garbage: - name = 'gc_rawrefcount_next_garbage' + elif self.instance is next_garbage_pyobj: + name = 'gc_rawrefcount_next_garbage_pyobj' hop.exception_cannot_occur() v_ob = hop.genop(name, [], resulttype = llmemory.Address) return _spec_ob(hop, v_ob) class Entry(ExtRegistryEntry): + _about_ = next_garbage_pypy + + def compute_result_annotation(self, s_Class): + from rpython.annotator import model as annmodel + assert s_Class.is_constant() + classdef = self.bookkeeper.getuniqueclassdef(s_Class.const) + return annmodel.SomeInstance(classdef, can_be_None=True) + + def specialize_call(self, hop): + hop.exception_cannot_occur() + v_p = hop.genop('gc_rawrefcount_next_garbage_pypy', [], + resulttype = llmemory.GCREF) + return _spec_p(hop, v_p) + +class Entry(ExtRegistryEntry): _about_ = cyclic_garbage_remove def compute_result_annotation(self): diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py --- a/rpython/rtyper/llinterp.py +++ b/rpython/rtyper/llinterp.py @@ -995,8 +995,10 @@ def op_gc_rawrefcount_cyclic_garbage_remove(self, *args): raise NotImplementedError("gc_rawrefcount_cyclic_garbage_remove") - def op_gc_rawrefcount_next_garbage(self, *args): - raise NotImplementedError("gc_rawrefcount_next_garbage") + def op_gc_rawrefcount_next_garbage_pypy(self, *args): + raise NotImplementedError("gc_rawrefcount_next_garbage_pypy") + def op_gc_rawrefcount_next_garbage_pyobj(self, *args): + raise NotImplementedError("gc_rawrefcount_next_garbage_pyobj") def op_do_malloc_fixedsize(self): raise NotImplementedError("do_malloc_fixedsize") diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -532,7 +532,8 @@ 'gc_rawrefcount_next_cyclic_isolate': LLOp(), 'gc_rawrefcount_cyclic_garbage_head': LLOp(sideeffects=False), 'gc_rawrefcount_cyclic_garbage_remove': LLOp(), - 'gc_rawrefcount_next_garbage': LLOp(), + 'gc_rawrefcount_next_garbage_pypy': LLOp(), + 'gc_rawrefcount_next_garbage_pyobj': LLOp(), 'gc_move_out_of_nursery': LLOp(), From pypy.commits at gmail.com Fri Mar 1 16:10:36 2019 From: pypy.commits at gmail.com (stevie_92) Date: Fri, 01 Mar 2019 13:10:36 -0800 (PST) Subject: [pypy-commit] pypy cpyext-gc-cycle: Started implementation for gc.garbage, adapted interface Message-ID: <5c799fcc.1c69fb81.38266.3968@mx.google.com> Author: Stefan Beyer Branch: cpyext-gc-cycle Changeset: r96198:77d74e85609f Date: 2019-03-01 22:09 +0100 http://bitbucket.org/pypy/pypy/changeset/77d74e85609f/ Log: Started implementation for gc.garbage, adapted interface diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py --- a/pypy/module/cpyext/state.py +++ b/pypy/module/cpyext/state.py @@ -13,7 +13,6 @@ # context. ExecutionContext.cpyext_operror = None - class State: def __init__(self, space): self.space = space @@ -80,8 +79,6 @@ from pypy.interpreter.baseobjspace import W_Root from pypy.module.cpyext.pyobject import PyObject, decref, \ incref, cts, finalize, from_ref - w_list = space.getattr(space.builtin_modules['gc'], - space.newtext('garbage')) print 'dealloc_trigger...' while True: ob = rawrefcount.next_dead(PyObject) @@ -116,17 +113,23 @@ if adr_int == llmemory.cast_adr_to_int( llmemory.cast_ptr_to_adr(head)): rawrefcount.cyclic_garbage_remove() + w_list = space.newlist([]) while True: w_obj = rawrefcount.next_garbage_pypy(W_Root) if not py_obj: break w_list.append(w_obj) + last_py_obj = lltype.nullptr(PyObject.TO) while True: - w_pyobj = rawrefcount.next_garbage_pyobj(PyObject) - if not py_obj: + w_pyobj = rawrefcount.next_garbage_pyobj(PyObject, + last_py_obj) + if not w_pyobj: break w_obj = from_ref(space, w_pyobj) w_list.append(w_obj) + last_py_obj = w_pyobj + space.setattr(space.builtin_modules['gc'], + space.newtext('garbage'), w_list) print 'dealloc_trigger DONE' return "RETRY" def tp_traverse(pyobj_ptr, callback, args): @@ -292,9 +295,6 @@ from pypy.module.cpyext.pyobject import (PyObject, incref, decref, finalize, from_ref) - w_list = space.getattr(space.builtin_modules['gc'], - space.newtext('garbage')) - while True: py_obj = rawrefcount.next_dead(PyObject) if not py_obj: @@ -321,17 +321,22 @@ if adr_int == llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(head)): rawrefcount.cyclic_garbage_remove() + w_list = space.newlist([]) while True: w_obj = rawrefcount.next_garbage_pypy(W_Root) if not py_obj: break w_list.append(w_obj) + last_py_obj = lltype.nullptr(PyObject.TO) while True: - w_pyobj = rawrefcount.next_garbage_pyobj(PyObject) - if not py_obj: + w_pyobj = rawrefcount.next_garbage_pyobj(PyObject, last_py_obj) + if not w_pyobj: break w_obj = from_ref(space, w_pyobj) w_list.append(w_obj) + last_py_obj = w_pyobj + space.setattr(space.builtin_modules['gc'], space.newtext('garbage'), + w_list) class PyObjDeallocAction(executioncontext.AsyncAction): """An action that invokes _Py_Dealloc() on the dying PyObjects. diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -3063,6 +3063,16 @@ rrc_enabled = False + # The default state. Here cyclic garbage with legacy finalizers is marked. + RAWREFCOUNT_STATE_DEFAULT = 0 + + # The state in which cyclic garbage with legacy finalizers is traced. + # Do not mark objects during this state, because we remove the flag + # during tracing and we do not want to trace those objects again. Also + # during this phase no new objects can be marked, as we are only building + # the list of cyclic garbage. + RAWREFCOUNT_STATE_GARBAGE = 1 + _ADDRARRAY = lltype.Array(llmemory.Address, hints={'nolength': True}) PYOBJ_HDR = lltype.Struct('GCHdr_PyObject', ('c_ob_refcnt', lltype.Signed), @@ -3128,6 +3138,7 @@ self.rrc_pyobj_as_gc = pyobj_as_gc self.rrc_finalizer_type = finalizer_type self.rrc_enabled = True + self.rrc_state = self.RAWREFCOUNT_STATE_DEFAULT def check_no_more_rawrefcount_state(self): "NOT_RPYTHON: for tests" @@ -3218,21 +3229,24 @@ next.c_gc_prev = gchdr def rawrefcount_next_garbage_pypy(self): + if self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT: + self.rrc_state = self.RAWREFCOUNT_STATE_GARBAGE + # return the next pypy object which is only reachable from garbage # pyobjects, probably need two more colors for this. one for marking # so that they stay alive during sweep, one for marking, so they do not # get returned here again - return lltype.nullptr(llmemory.GCREF.TO) - - def rawrefcount_next_garbage_pyobj(self): + result = lltype.nullptr(llmemory.GCREF.TO) + + if result == lltype.nullptr(llmemory.GCREF.TO): + self.rrc_state = self.RAWREFCOUNT_STATE_DEFAULT + return result + + def rawrefcount_next_garbage_pyobj(self, curr_pyobj): # implement st objects in this list still remain in the set of # all pyobjs, because references could still change and cause them # to live again. also keep in mind, that state will create references # to pyobjs in this list and might increment the refcount. - - # use create_link_pyobj on the result to create gc objects for pyobjects - #p = W_Root(42) - #p.pyobj = ob return llmemory.NULL diff --git a/rpython/memory/gc/test/test_rawrefcount.py b/rpython/memory/gc/test/test_rawrefcount.py --- a/rpython/memory/gc/test/test_rawrefcount.py +++ b/rpython/memory/gc/test/test_rawrefcount.py @@ -627,12 +627,16 @@ self.gc.rawrefcount_cyclic_garbage_remove() next_dead = self.gc.rawrefcount_cyclic_garbage_head() - next = self.gc.rawrefcount_next_garbage_pypy() - while next <> lltype.nullptr(llmemory.GCREF.TO): - garbage_pypy.append(next) - next = self.gc.rawrefcount_next_garbage_pyobj() + next_garbage = self.gc.rawrefcount_next_garbage_pypy() + while next_garbage <> lltype.nullptr(llmemory.GCREF.TO): + garbage_pypy.append(next_garbage) + next_garbage = self.gc.rawrefcount_next_garbage_pypy() + last_pyobj = llmemory.NULL + next = self.gc.rawrefcount_next_garbage_pyobj(last_pyobj) while next <> llmemory.NULL: garbage_pyobj.append(next) + next = self.gc.rawrefcount_next_garbage_pyobj(last_pyobj) + last_pyobj = next # do a collection to find cyclic isolates and clean them, if there are # no finalizers diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -522,7 +522,7 @@ GCClass.rawrefcount_next_garbage_pypy, [s_gc], s_gcref, inline = True) self.rawrefcount_next_garbage_pyobj_ptr = getfn( - GCClass.rawrefcount_next_garbage_pyobj, [s_gc], + GCClass.rawrefcount_next_garbage_pyobj, [s_gc, SomeAddress()], SomeAddress(), inline = True) if GCClass.can_usually_pin_objects: @@ -1439,9 +1439,12 @@ resultvar=hop.spaceop.result) def gct_gc_rawrefcount_next_garbage_pyobj(self, hop): + [v_pyobject] = hop.spaceop.args + assert v_pyobject.concretetype == llmemory.Address assert hop.spaceop.result.concretetype == llmemory.Address hop.genop("direct_call", - [self.rawrefcount_next_garbage_pyobj_ptr, self.c_const_gc], + [self.rawrefcount_next_garbage_pyobj_ptr, self.c_const_gc, + v_pyobject], resultvar=hop.spaceop.result) def _set_into_gc_array_part(self, op): diff --git a/rpython/rlib/rawrefcount.py b/rpython/rlib/rawrefcount.py --- a/rpython/rlib/rawrefcount.py +++ b/rpython/rlib/rawrefcount.py @@ -147,10 +147,10 @@ @not_rpython def next_garbage_pypy(Class): - return lltype.nullptr(llmemory.GCREF.TO) + return None @not_rpython -def next_garbage_pyobj(OB_PTR_TYPE): +def next_garbage_pyobj(OB_PTR_TYPE, curr_pyobj): return lltype.nullptr(OB_PTR_TYPE.TO) @not_rpython @@ -375,8 +375,7 @@ return _spec_p(hop, v_p) class Entry(ExtRegistryEntry): - _about_ = (next_dead, cyclic_garbage_head, next_cyclic_isolate, - next_garbage_pyobj) + _about_ = (next_dead, cyclic_garbage_head, next_cyclic_isolate) def compute_result_annotation(self, s_OB_PTR_TYPE): from rpython.rtyper.llannotation import lltype_to_annotation @@ -390,13 +389,28 @@ name = 'gc_rawrefcount_cyclic_garbage_head' elif self.instance is next_cyclic_isolate: name = 'gc_rawrefcount_next_cyclic_isolate' - elif self.instance is next_garbage_pyobj: - name = 'gc_rawrefcount_next_garbage_pyobj' hop.exception_cannot_occur() v_ob = hop.genop(name, [], resulttype = llmemory.Address) return _spec_ob(hop, v_ob) class Entry(ExtRegistryEntry): + _about_ = next_garbage_pyobj + + def compute_result_annotation(self, s_OB_PTR_TYPE, s_ob): + from rpython.rtyper.llannotation import lltype_to_annotation, SomePtr + assert s_OB_PTR_TYPE.is_constant() + assert isinstance(s_ob, SomePtr) + return lltype_to_annotation(s_OB_PTR_TYPE.const) + + def specialize_call(self, hop): + hop.exception_cannot_occur() + v_ob = hop.inputarg(hop.args_r[1], arg=1) + v_ob_res = hop.genop('gc_rawrefcount_next_garbage_pyobj', + [_unspec_ob(hop, v_ob)], + resulttype = llmemory.Address) + return _spec_ob(hop, v_ob_res) + +class Entry(ExtRegistryEntry): _about_ = next_garbage_pypy def compute_result_annotation(self, s_Class): From pypy.commits at gmail.com Sat Mar 2 08:19:38 2019 From: pypy.commits at gmail.com (stevie_92) Date: Sat, 02 Mar 2019 05:19:38 -0800 (PST) Subject: [pypy-commit] pypy cpyext-gc-cycle: Implemented additional rawrefcount states and added a gc-header flag Message-ID: <5c7a82ea.1c69fb81.ee144.2a97@mx.google.com> Author: Stefan Beyer Branch: cpyext-gc-cycle Changeset: r96199:f0fce8ff6d08 Date: 2019-03-02 11:15 +0100 http://bitbucket.org/pypy/pypy/changeset/f0fce8ff6d08/ Log: Implemented additional rawrefcount states and added a gc-header flag Extended interface diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py --- a/pypy/module/cpyext/state.py +++ b/pypy/module/cpyext/state.py @@ -113,6 +113,7 @@ if adr_int == llmemory.cast_adr_to_int( llmemory.cast_ptr_to_adr(head)): rawrefcount.cyclic_garbage_remove() + rawrefcount.begin_garbage() w_list = space.newlist([]) while True: w_obj = rawrefcount.next_garbage_pypy(W_Root) @@ -130,6 +131,7 @@ last_py_obj = w_pyobj space.setattr(space.builtin_modules['gc'], space.newtext('garbage'), w_list) + rawrefcount.end_garbage() print 'dealloc_trigger DONE' return "RETRY" def tp_traverse(pyobj_ptr, callback, args): @@ -321,6 +323,7 @@ if adr_int == llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(head)): rawrefcount.cyclic_garbage_remove() + rawrefcount.begin_garbage() w_list = space.newlist([]) while True: w_obj = rawrefcount.next_garbage_pypy(W_Root) @@ -337,6 +340,7 @@ last_py_obj = w_pyobj space.setattr(space.builtin_modules['gc'], space.newtext('garbage'), w_list) + rawrefcount.end_garbage() class PyObjDeallocAction(executioncontext.AsyncAction): """An action that invokes _Py_Dealloc() on the dying PyObjects. diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -163,7 +163,15 @@ # It does not need an additional copy in trace out GCFLAG_SHADOW_INITIALIZED = first_gcflag << 11 -_GCFLAG_FIRST_UNUSED = first_gcflag << 12 # the first unused bit +# Objects referenced only from legacy rawrefcount finalizers which have +# not been added to gc.garbage have this flag set. It is set during the +# rawrefcount marking phase and removed once the object is returned to +# the caller, who is adding it to the gc.garbage list. During the next +# collection cycle this process is repeated, as the set of objects might +# have changed. +GCFLAG_GARBAGE = first_gcflag << 12 + +_GCFLAG_FIRST_UNUSED = first_gcflag << 13 # the first unused bit # States for the incremental GC @@ -2711,6 +2719,9 @@ # to also set TRACK_YOUNG_PTRS here, for the write barrier. hdr.tid |= GCFLAG_VISITED | GCFLAG_TRACK_YOUNG_PTRS + if self.rrc_state == self.RAWREFCOUNT_STATE_MARKING: + hdr.tid |= GCFLAG_GARBAGE + if self.has_gcptr(llop.extract_ushort(llgroup.HALFWORD, hdr.tid)): # # Trace the content of the object and put all objects it references @@ -3063,15 +3074,18 @@ rrc_enabled = False - # The default state. Here cyclic garbage with legacy finalizers is marked. + # Default state, no rawrefcount specific code is executed during normal marking. RAWREFCOUNT_STATE_DEFAULT = 0 + # Here cyclic garbage only reachable from legacy finalizers is marked. + RAWREFCOUNT_STATE_MARKING = 1 + # The state in which cyclic garbage with legacy finalizers is traced. # Do not mark objects during this state, because we remove the flag # during tracing and we do not want to trace those objects again. Also # during this phase no new objects can be marked, as we are only building # the list of cyclic garbage. - RAWREFCOUNT_STATE_GARBAGE = 1 + RAWREFCOUNT_STATE_GARBAGE = 2 _ADDRARRAY = lltype.Array(llmemory.Address, hints={'nolength': True}) PYOBJ_HDR = lltype.Struct('GCHdr_PyObject', @@ -3228,19 +3242,22 @@ gchdr.c_gc_next = next next.c_gc_prev = gchdr + def rawrefcount_begin_garbage(self): + # after this, no new objects should be marked with the GCFLAG_GARBAGE + # flag + self.rrc_state = self.RAWREFCOUNT_STATE_GARBAGE + + def rawrefcount_end_garbage(self): + # now there should not be any object left with a GCFLAG_GARBAGE flag + # set + self.rrc_state = self.RAWREFCOUNT_STATE_DEFAULT + def rawrefcount_next_garbage_pypy(self): - if self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT: - self.rrc_state = self.RAWREFCOUNT_STATE_GARBAGE - # return the next pypy object which is only reachable from garbage # pyobjects, probably need two more colors for this. one for marking # so that they stay alive during sweep, one for marking, so they do not # get returned here again - result = lltype.nullptr(llmemory.GCREF.TO) - - if result == lltype.nullptr(llmemory.GCREF.TO): - self.rrc_state = self.RAWREFCOUNT_STATE_DEFAULT - return result + return lltype.nullptr(llmemory.GCREF.TO) def rawrefcount_next_garbage_pyobj(self, curr_pyobj): # implement st objects in this list still remain in the set of @@ -3249,7 +3266,6 @@ # to pyobjs in this list and might increment the refcount. return llmemory.NULL - def rrc_invoke_callback(self): if self.rrc_enabled and (self.rrc_dealloc_pending.non_empty() or self.rrc_pyobj_isolate_list.c_gc_next <> @@ -3365,61 +3381,63 @@ _rrc_free._always_inline_ = True def rrc_major_collection_trace(self): - if not self._rrc_gc_list_is_empty(self.rrc_pyobj_old_list): - # Check, if the cyclic isolate from the last collection cycle - # is reachable from outside, after the finalizers have been - # executed. - self._rrc_collect_rawrefcount_roots(self.rrc_pyobj_old_list) - found_alive = False + if self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT: + if not self._rrc_gc_list_is_empty(self.rrc_pyobj_old_list): + # Check, if the cyclic isolate from the last collection cycle + # is reachable from outside, after the finalizers have been + # executed. + self._rrc_collect_rawrefcount_roots(self.rrc_pyobj_old_list) + found_alive = False + gchdr = self.rrc_pyobj_old_list.c_gc_next + while gchdr <> self.rrc_pyobj_old_list: + if (gchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT) > 0: + found_alive = True + break + gchdr = gchdr.c_gc_next + if found_alive: + self._rrc_gc_list_merge(self.rrc_pyobj_old_list, + self.rrc_pyobj_list) + else: + self._rrc_gc_list_merge(self.rrc_pyobj_old_list, + self.rrc_pyobj_garbage_list) + + self._rrc_collect_rawrefcount_roots(self.rrc_pyobj_list) + self._rrc_mark_rawrefcount() + + # handle legacy finalizers + self.rrc_state = self.RAWREFCOUNT_STATE_MARKING + # TODO: move all pyobjs to sublist of pyobj_list and reset cyclic refcount + # TODO: mark all pypy-objects with special flag and mark them visited + self.rrc_state = self.RAWREFCOUNT_STATE_DEFAULT + + # now handle modern finalizers + found_finalizer = False gchdr = self.rrc_pyobj_old_list.c_gc_next while gchdr <> self.rrc_pyobj_old_list: - if (gchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT) > 0: - found_alive = True - break + if self.rrc_finalizer_type(gchdr) == \ + self.RAWREFCOUNT_FINALIZER_MODERN: + found_finalizer = True gchdr = gchdr.c_gc_next - if found_alive: - self._rrc_gc_list_merge(self.rrc_pyobj_old_list, - self.rrc_pyobj_list) - else: - self._rrc_gc_list_merge(self.rrc_pyobj_old_list, - self.rrc_pyobj_garbage_list) - - self._rrc_collect_rawrefcount_roots(self.rrc_pyobj_list) - self._rrc_mark_rawrefcount() - - found_finalizer = False - gchdr = self.rrc_pyobj_old_list.c_gc_next - while gchdr <> self.rrc_pyobj_old_list: - if self.rrc_finalizer_type(gchdr) == \ - self.RAWREFCOUNT_FINALIZER_MODERN: - found_finalizer = True - gchdr = gchdr.c_gc_next - if found_finalizer: - self._rrc_gc_list_move(self.rrc_pyobj_old_list, - self.rrc_pyobj_isolate_list) - - self.rrc_p_list_old.foreach(self._rrc_major_trace, found_finalizer) - self.rrc_o_list_old.foreach(self._rrc_major_trace, found_finalizer) - - # TODO: for all unreachable objects, which are marked potentially - # TODO: uncollectable, move them to the set of uncollectable objs - - # TODO: for all unreachable objects with tp_del (legacy finalizer), - # TODO: except for border objects with a refcount of - # TODO: REFCNT_FROM_PYPY (equals zero at this point): - # TODO: * mark reachable pypy objects - # TODO: * move reachable cpython objects back to pyobj_list - # TODO: * mark all reachable objects as potentially uncollectable + if found_finalizer: + self._rrc_gc_list_move(self.rrc_pyobj_old_list, + self.rrc_pyobj_isolate_list) + + use_cylicrefcnt = not found_finalizer + else: + use_cylicrefcnt = False + + self.rrc_p_list_old.foreach(self._rrc_major_trace, use_cylicrefcnt) + self.rrc_o_list_old.foreach(self._rrc_major_trace, use_cylicrefcnt) # TODO: handle weakrefs for unreachable objects and create # TODO: a list of callbacks, which has to be called after the # TODO: the GC runs - def _rrc_major_trace(self, pyobject, found_finalizer): + def _rrc_major_trace(self, pyobject, use_cylicrefcnt): from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT # - if not found_finalizer: + if use_cylicrefcnt: pygchdr = self.rrc_pyobj_as_gc(self._pyobj(pyobject)) if pygchdr != lltype.nullptr(self.PYOBJ_GC_HDR): rc = pygchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT @@ -3457,9 +3475,10 @@ self.rrc_o_list_old.delete() self.rrc_o_list_old = new_o_list - if not self._rrc_gc_list_is_empty(self.rrc_pyobj_old_list): - self._rrc_gc_list_merge(self.rrc_pyobj_old_list, - self.rrc_pyobj_garbage_list) + if self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT: + if not self._rrc_gc_list_is_empty(self.rrc_pyobj_old_list): + self._rrc_gc_list_merge(self.rrc_pyobj_old_list, + self.rrc_pyobj_garbage_list) def _rrc_major_free(self, pyobject, surviving_list, surviving_dict): # The pyobject survives if the corresponding obj survives. diff --git a/rpython/memory/gc/test/test_rawrefcount.py b/rpython/memory/gc/test/test_rawrefcount.py --- a/rpython/memory/gc/test/test_rawrefcount.py +++ b/rpython/memory/gc/test/test_rawrefcount.py @@ -627,6 +627,7 @@ self.gc.rawrefcount_cyclic_garbage_remove() next_dead = self.gc.rawrefcount_cyclic_garbage_head() + self.gc.rawrefcount_begin_garbage() next_garbage = self.gc.rawrefcount_next_garbage_pypy() while next_garbage <> lltype.nullptr(llmemory.GCREF.TO): garbage_pypy.append(next_garbage) @@ -637,6 +638,7 @@ garbage_pyobj.append(next) next = self.gc.rawrefcount_next_garbage_pyobj(last_pyobj) last_pyobj = next + self.gc.rawrefcount_end_garbage() # do a collection to find cyclic isolates and clean them, if there are # no finalizers diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -518,6 +518,12 @@ self.rawrefcount_cyclic_garbage_remove_ptr = getfn( GCClass.rawrefcount_cyclic_garbage_remove, [s_gc], annmodel.s_None, inline = True) + self.rawrefcount_begin_garbage_ptr = getfn( + GCClass.rawrefcount_begin_garbage, [s_gc], annmodel.s_None, + inline = True) + self.rawrefcount_end_garbage_ptr = getfn( + GCClass.rawrefcount_end_garbage, [s_gc], annmodel.s_None, + inline = True) self.rawrefcount_next_garbage_pypy_ptr = getfn( GCClass.rawrefcount_next_garbage_pypy, [s_gc], s_gcref, inline = True) @@ -1432,6 +1438,14 @@ [self.rawrefcount_cyclic_garbage_remove_ptr, self.c_const_gc]) + def gct_gc_rawrefcount_begin_garbage(self, hop): + hop.genop("direct_call", + [self.rawrefcount_begin_garbage_ptr, self.c_const_gc]) + + def gct_gc_rawrefcount_end_garbage(self, hop): + hop.genop("direct_call", + [self.rawrefcount_end_garbage_ptr, self.c_const_gc]) + def gct_gc_rawrefcount_next_garbage_pypy(self, hop): assert hop.spaceop.result.concretetype == llmemory.GCREF hop.genop("direct_call", diff --git a/rpython/rlib/rawrefcount.py b/rpython/rlib/rawrefcount.py --- a/rpython/rlib/rawrefcount.py +++ b/rpython/rlib/rawrefcount.py @@ -146,6 +146,14 @@ return lltype.nullptr(OB_PTR_TYPE.TO) @not_rpython +def begin_garbage(): + pass + + at not_rpython +def end_garbage(): + pass + + at not_rpython def next_garbage_pypy(Class): return None @@ -426,14 +434,20 @@ return _spec_p(hop, v_p) class Entry(ExtRegistryEntry): - _about_ = cyclic_garbage_remove + _about_ = (cyclic_garbage_remove, begin_garbage, end_garbage) def compute_result_annotation(self): pass def specialize_call(self, hop): hop.exception_cannot_occur() - hop.genop('gc_rawrefcount_cyclic_garbage_remove', []) + if self.instance is cyclic_garbage_remove: + name = 'gc_rawrefcount_cyclic_garbage_remove' + elif self.instance is begin_garbage: + name = 'gc_rawrefcount_begin_garbage' + elif self.instance is end_garbage: + name = 'gc_rawrefcount_end_garbage' + hop.genop(name, []) src_dir = py.path.local(__file__).dirpath() / 'src' boehm_eci = ExternalCompilationInfo( diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py --- a/rpython/rtyper/llinterp.py +++ b/rpython/rtyper/llinterp.py @@ -995,6 +995,10 @@ def op_gc_rawrefcount_cyclic_garbage_remove(self, *args): raise NotImplementedError("gc_rawrefcount_cyclic_garbage_remove") + def op_gc_rawrefcount_begin_garbage(self, *args): + raise NotImplementedError("gc_rawrefcount_begin_garbage") + def op_gc_rawrefcount_end_garbage(self, *args): + raise NotImplementedError("gc_rawrefcount_end_garbage") def op_gc_rawrefcount_next_garbage_pypy(self, *args): raise NotImplementedError("gc_rawrefcount_next_garbage_pypy") def op_gc_rawrefcount_next_garbage_pyobj(self, *args): diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -532,6 +532,8 @@ 'gc_rawrefcount_next_cyclic_isolate': LLOp(), 'gc_rawrefcount_cyclic_garbage_head': LLOp(sideeffects=False), 'gc_rawrefcount_cyclic_garbage_remove': LLOp(), + 'gc_rawrefcount_begin_garbage': LLOp(), + 'gc_rawrefcount_end_garbage': LLOp(), 'gc_rawrefcount_next_garbage_pypy': LLOp(), 'gc_rawrefcount_next_garbage_pyobj': LLOp(), From pypy.commits at gmail.com Sat Mar 2 08:46:28 2019 From: pypy.commits at gmail.com (stevie_92) Date: Sat, 02 Mar 2019 05:46:28 -0800 (PST) Subject: [pypy-commit] pypy cpyext-gc-cycle: Fixed issue in gc if rawrefcount is disabled Message-ID: <5c7a8934.1c69fb81.17647.b93d@mx.google.com> Author: Stefan Beyer Branch: cpyext-gc-cycle Changeset: r96200:85b1a76a7d3a Date: 2019-03-02 14:45 +0100 http://bitbucket.org/pypy/pypy/changeset/85b1a76a7d3a/ Log: Fixed issue in gc if rawrefcount is disabled Refactored rrc code to make it easier to read diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -2719,7 +2719,8 @@ # to also set TRACK_YOUNG_PTRS here, for the write barrier. hdr.tid |= GCFLAG_VISITED | GCFLAG_TRACK_YOUNG_PTRS - if self.rrc_state == self.RAWREFCOUNT_STATE_MARKING: + if self.rrc_enabled and \ + self.rrc_state == self.RAWREFCOUNT_STATE_MARKING: hdr.tid |= GCFLAG_GARBAGE if self.has_gcptr(llop.extract_ushort(llgroup.HALFWORD, hdr.tid)): @@ -3381,53 +3382,24 @@ _rrc_free._always_inline_ = True def rrc_major_collection_trace(self): + # Only trace and mark rawrefcounted object if we are not doing + # something special, like building gc.garbage. if self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT: + # check objects with finalizers from last collection cycle if not self._rrc_gc_list_is_empty(self.rrc_pyobj_old_list): - # Check, if the cyclic isolate from the last collection cycle - # is reachable from outside, after the finalizers have been - # executed. - self._rrc_collect_rawrefcount_roots(self.rrc_pyobj_old_list) - found_alive = False - gchdr = self.rrc_pyobj_old_list.c_gc_next - while gchdr <> self.rrc_pyobj_old_list: - if (gchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT) > 0: - found_alive = True - break - gchdr = gchdr.c_gc_next - if found_alive: - self._rrc_gc_list_merge(self.rrc_pyobj_old_list, - self.rrc_pyobj_list) - else: - self._rrc_gc_list_merge(self.rrc_pyobj_old_list, - self.rrc_pyobj_garbage_list) - + self._rrc_check_finalizer() + # collect all rawrefcounted roots self._rrc_collect_rawrefcount_roots(self.rrc_pyobj_list) + # mark all objects reachable from rawrefcounted roots self._rrc_mark_rawrefcount() - - # handle legacy finalizers - self.rrc_state = self.RAWREFCOUNT_STATE_MARKING - # TODO: move all pyobjs to sublist of pyobj_list and reset cyclic refcount - # TODO: mark all pypy-objects with special flag and mark them visited - self.rrc_state = self.RAWREFCOUNT_STATE_DEFAULT - - # now handle modern finalizers - found_finalizer = False - gchdr = self.rrc_pyobj_old_list.c_gc_next - while gchdr <> self.rrc_pyobj_old_list: - if self.rrc_finalizer_type(gchdr) == \ - self.RAWREFCOUNT_FINALIZER_MODERN: - found_finalizer = True - gchdr = gchdr.c_gc_next - if found_finalizer: - self._rrc_gc_list_move(self.rrc_pyobj_old_list, - self.rrc_pyobj_isolate_list) - - use_cylicrefcnt = not found_finalizer + self._rrc_mark_garbage() # handle legacy finalizers + use_cylicrc = not self._rrc_find_finalizer() # modern finalizers else: - use_cylicrefcnt = False - - self.rrc_p_list_old.foreach(self._rrc_major_trace, use_cylicrefcnt) - self.rrc_o_list_old.foreach(self._rrc_major_trace, use_cylicrefcnt) + use_cylicrc = False # don't sweep any objects in cyclic isolates + + # now mark all pypy objects at the border, depending on the results + self.rrc_p_list_old.foreach(self._rrc_major_trace, use_cylicrc) + self.rrc_o_list_old.foreach(self._rrc_major_trace, use_cylicrc) # TODO: handle weakrefs for unreachable objects and create # TODO: a list of callbacks, which has to be called after the @@ -3581,6 +3553,44 @@ # now all rawrefcounted objects, which are alive, have a cyclic # refcount > 0 or are marked + def _rrc_mark_garbage(self): + self.rrc_state = self.RAWREFCOUNT_STATE_MARKING + # TODO: move all pyobjs to sublist of pyobj_list and reset cyclic refcount + # TODO: mark all pypy-objects with special flag and mark them visited + self.rrc_state = self.RAWREFCOUNT_STATE_DEFAULT + + def _rrc_check_finalizer(self): + # Check, if the cyclic isolate from the last collection cycle + # is reachable from outside, after the finalizers have been + # executed. + self._rrc_collect_rawrefcount_roots(self.rrc_pyobj_old_list) + found_alive = False + gchdr = self.rrc_pyobj_old_list.c_gc_next + while gchdr <> self.rrc_pyobj_old_list: + if (gchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT) > 0: + found_alive = True + break + gchdr = gchdr.c_gc_next + if found_alive: + self._rrc_gc_list_merge(self.rrc_pyobj_old_list, + self.rrc_pyobj_list) + else: + self._rrc_gc_list_merge(self.rrc_pyobj_old_list, + self.rrc_pyobj_garbage_list) + + def _rrc_find_finalizer(self): + found_finalizer = False + gchdr = self.rrc_pyobj_old_list.c_gc_next + while gchdr <> self.rrc_pyobj_old_list: + if self.rrc_finalizer_type(gchdr) == \ + self.RAWREFCOUNT_FINALIZER_MODERN: + found_finalizer = True + gchdr = gchdr.c_gc_next + if found_finalizer: + self._rrc_gc_list_move(self.rrc_pyobj_old_list, + self.rrc_pyobj_isolate_list) + return found_finalizer + def _rrc_visit(pyobj, self_ptr): from rpython.rtyper.annlowlevel import cast_adr_to_nongc_instance # From pypy.commits at gmail.com Sun Mar 3 03:24:31 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 03 Mar 2019 00:24:31 -0800 (PST) Subject: [pypy-commit] pypy py3.6: FormatErrorW returns a tuple text, unicode_len Message-ID: <5c7b8f3f.1c69fb81.500ef.bd82@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r96201:fe52877bd963 Date: 2019-03-03 10:26 +0200 http://bitbucket.org/pypy/pypy/changeset/fe52877bd963/ Log: FormatErrorW returns a tuple text, unicode_len diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py --- a/pypy/module/_winreg/interp_winreg.py +++ b/pypy/module/_winreg/interp_winreg.py @@ -10,9 +10,9 @@ def raiseWindowsError(space, errcode, context): message = rwin32.FormatErrorW(errcode) w_errcode = space.newint(errcode) - raise OperationError(space.w_WindowsError, - space.newtuple([w_errcode, space.newtext(message), - space.w_None, w_errcode])) + w_t = space.newtuple([w_errcode, space.newtext(*message), + space.w_None, w_errcode]) + raise OperationError(space.w_WindowsError, w_t) class W_HKEY(W_Root): def __init__(self, space, hkey): From pypy.commits at gmail.com Sun Mar 3 05:58:11 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 03 Mar 2019 02:58:11 -0800 (PST) Subject: [pypy-commit] pypy py3.6: fix 323ddcb8a8c4 for win32 which needs to define the macro, sse2 is always supported Message-ID: <5c7bb343.1c69fb81.1557b.b650@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r96204:28ca4c3a2982 Date: 2019-03-03 13:00 +0200 http://bitbucket.org/pypy/pypy/changeset/28ca4c3a2982/ Log: fix 323ddcb8a8c4 for win32 which needs to define the macro, sse2 is always supported diff --git a/lib_pypy/_blake2/_blake2_build.py b/lib_pypy/_blake2/_blake2_build.py --- a/lib_pypy/_blake2/_blake2_build.py +++ b/lib_pypy/_blake2/_blake2_build.py @@ -8,11 +8,14 @@ IS_WIN = sys.platform == 'win32' if IS_ARM: # XXX Choose neon accelaration + define_macros = [] extra_compile_args = [] elif IS_WIN: - extra_compile_args = ['/arch:SSE2'] + extra_compile_args = [] + define_macros = [('__SSE2__', '1')] else: extra_compile_args = ['-msse2'] + define_macros = [] @@ -86,6 +89,7 @@ ], include_dirs=[_libdir], extra_compile_args=extra_compile_args, + define_macros=define_macros, ) def _replace_b2s(src): @@ -102,6 +106,7 @@ ], include_dirs=[_libdir], extra_compile_args=extra_compile_args, + define_macros=define_macros, ) if __name__ == '__main__': From pypy.commits at gmail.com Sun Mar 3 12:54:36 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 03 Mar 2019 09:54:36 -0800 (PST) Subject: [pypy-commit] pypy py3.6: FormatErrorW returns a tuple text, unicode_len Message-ID: <5c7c14dc.1c69fb81.bc30d.f1f5@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r96205:ce6ce17c04ae Date: 2019-03-03 19:56 +0200 http://bitbucket.org/pypy/pypy/changeset/ce6ce17c04ae/ Log: FormatErrorW returns a tuple text, unicode_len diff --git a/pypy/module/_cffi_backend/cerrno.py b/pypy/module/_cffi_backend/cerrno.py --- a/pypy/module/_cffi_backend/cerrno.py +++ b/pypy/module/_cffi_backend/cerrno.py @@ -27,4 +27,4 @@ if code == -1: code = GetLastError_alt_saved() message = FormatErrorW(code) - return space.newtuple([space.newint(code), space.newtext(message)]) + return space.newtuple([space.newint(code), space.newtext(*message)]) diff --git a/pypy/module/_multiprocessing/interp_win32_py3.py b/pypy/module/_multiprocessing/interp_win32_py3.py --- a/pypy/module/_multiprocessing/interp_win32_py3.py +++ b/pypy/module/_multiprocessing/interp_win32_py3.py @@ -9,7 +9,7 @@ message = rwin32.FormatErrorW(errno) w_errcode = space.newint(errno) return OperationError(space.w_WindowsError, - space.newtuple([w_errcode, space.newtext(message), + space.newtuple([w_errcode, space.newtext(*message), space.w_None, w_errcode])) @unwrap_spec(handle=int) diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py --- a/pypy/module/_rawffi/interp_rawffi.py +++ b/pypy/module/_rawffi/interp_rawffi.py @@ -632,7 +632,7 @@ if _MS_WINDOWS: @unwrap_spec(code=int) def FormatError(space, code): - return space.newtext(rwin32.FormatErrorW(code)) + return space.newtext(*rwin32.FormatErrorW(code)) @unwrap_spec(hresult=int) def check_HRESULT(space, hresult): From pypy.commits at gmail.com Sun Mar 3 15:21:57 2019 From: pypy.commits at gmail.com (stevie_92) Date: Sun, 03 Mar 2019 12:21:57 -0800 (PST) Subject: [pypy-commit] pypy cpyext-gc-cycle: First version of rawrefcount legacy finalizer implementation Message-ID: <5c7c3765.1c69fb81.52fc1.fd7e@mx.google.com> Author: Stefan Beyer Branch: cpyext-gc-cycle Changeset: r96206:3ab559b9b893 Date: 2019-03-03 21:21 +0100 http://bitbucket.org/pypy/pypy/changeset/3ab559b9b893/ Log: First version of rawrefcount legacy finalizer implementation Test is still failing, because garbage_pypy is not implemented yet Adapted interface diff --git a/pypy/module/cpyext/src/object.c b/pypy/module/cpyext/src/object.c --- a/pypy/module/cpyext/src/object.c +++ b/pypy/module/cpyext/src/object.c @@ -53,14 +53,15 @@ _PyPy_finalizer_type(PyGC_Head *gc) { PyObject *op = FROM_GC(gc); - if (!_PyGCHead_FINALIZED(gc) && + if (Py_TYPE(op)->tp_del != NULL) { + return 2; // legacy (has priority over modern) + } else if (!_PyGCHead_FINALIZED(gc) && PyType_HasFeature(Py_TYPE(op), Py_TPFLAGS_HAVE_FINALIZE) && Py_TYPE(op)->tp_finalize != NULL) { - return 1; + return 1; // modern } else { - return 0; + return 0; // no finalizer } - // TODO: legacy finalizer (tp_del) } void diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py --- a/pypy/module/cpyext/state.py +++ b/pypy/module/cpyext/state.py @@ -120,15 +120,12 @@ if not py_obj: break w_list.append(w_obj) - last_py_obj = lltype.nullptr(PyObject.TO) while True: - w_pyobj = rawrefcount.next_garbage_pyobj(PyObject, - last_py_obj) + w_pyobj = rawrefcount.next_garbage_pyobj(PyObject) if not w_pyobj: break w_obj = from_ref(space, w_pyobj) w_list.append(w_obj) - last_py_obj = w_pyobj space.setattr(space.builtin_modules['gc'], space.newtext('garbage'), w_list) rawrefcount.end_garbage() @@ -330,14 +327,12 @@ if not py_obj: break w_list.append(w_obj) - last_py_obj = lltype.nullptr(PyObject.TO) while True: - w_pyobj = rawrefcount.next_garbage_pyobj(PyObject, last_py_obj) + w_pyobj = rawrefcount.next_garbage_pyobj(PyObject) if not w_pyobj: break w_obj = from_ref(space, w_pyobj) w_list.append(w_obj) - last_py_obj = w_pyobj space.setattr(space.builtin_modules['gc'], space.newtext('garbage'), w_list) rawrefcount.end_garbage() diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -3137,18 +3137,10 @@ self.rrc_dealloc_pending = self.AddressStack() self.rrc_tp_traverse = tp_traverse self.rrc_pyobj_list = self._pygchdr(pyobj_list) - self.rrc_pyobj_old_list = \ - lltype.malloc(self.PYOBJ_GC_HDR, flavor='raw', immortal=True) - self.rrc_pyobj_old_list.c_gc_next = self.rrc_pyobj_old_list - self.rrc_pyobj_old_list.c_gc_prev = self.rrc_pyobj_old_list - self.rrc_pyobj_isolate_list = \ - lltype.malloc(self.PYOBJ_GC_HDR, flavor='raw', immortal=True) - self.rrc_pyobj_isolate_list.c_gc_next = self.rrc_pyobj_isolate_list - self.rrc_pyobj_isolate_list.c_gc_prev = self.rrc_pyobj_isolate_list - self.rrc_pyobj_garbage_list = \ - lltype.malloc(self.PYOBJ_GC_HDR, flavor='raw', immortal=True) - self.rrc_pyobj_garbage_list.c_gc_next = self.rrc_pyobj_garbage_list - self.rrc_pyobj_garbage_list.c_gc_prev = self.rrc_pyobj_garbage_list + self.rrc_pyobj_old_list = self._rrc_gc_list_new() + self.rrc_pyobj_isolate_list = self._rrc_gc_list_new() + self.rrc_pyobj_dead_list = self._rrc_gc_list_new() + self.rrc_pyobj_garbage_list = self._rrc_gc_list_new() self.rrc_gc_as_pyobj = gc_as_pyobj self.rrc_pyobj_as_gc = pyobj_as_gc self.rrc_finalizer_type = finalizer_type @@ -3224,14 +3216,14 @@ return llmemory.NULL def rawrefcount_cyclic_garbage_head(self): - if not self._rrc_gc_list_is_empty(self.rrc_pyobj_garbage_list): + if not self._rrc_gc_list_is_empty(self.rrc_pyobj_dead_list): return llmemory.cast_ptr_to_adr( - self.rrc_gc_as_pyobj(self.rrc_pyobj_garbage_list.c_gc_next)) + self.rrc_gc_as_pyobj(self.rrc_pyobj_dead_list.c_gc_next)) else: return llmemory.NULL def rawrefcount_cyclic_garbage_remove(self): - gchdr = self.rrc_pyobj_garbage_list.c_gc_next + gchdr = self.rrc_pyobj_dead_list.c_gc_next # remove from old list next = gchdr.c_gc_next next.c_gc_prev = gchdr.c_gc_prev @@ -3254,25 +3246,40 @@ self.rrc_state = self.RAWREFCOUNT_STATE_DEFAULT def rawrefcount_next_garbage_pypy(self): - # return the next pypy object which is only reachable from garbage - # pyobjects, probably need two more colors for this. one for marking - # so that they stay alive during sweep, one for marking, so they do not - # get returned here again - return lltype.nullptr(llmemory.GCREF.TO) - - def rawrefcount_next_garbage_pyobj(self, curr_pyobj): - # implement st objects in this list still remain in the set of - # all pyobjs, because references could still change and cause them - # to live again. also keep in mind, that state will create references - # to pyobjs in this list and might increment the refcount. - return llmemory.NULL + # We assume that next_garbage_pypy is always called before + # next_garbage_pyobj. As pypy objects can only be in garbage, if there + # is at least one pyobj in garbage, we can use this optimization. + if self._rrc_gc_list_is_empty(self.rrc_pyobj_garbage_list): + return lltype.nullptr(llmemory.GCREF.TO) + else: + # TODO: return the next pypy object which is marked with GCFLAG_GARBAGE and + # remove the flag from this object. We can safely assume that objects + # do not move, as this can only happen to old objects. + return lltype.nullptr(llmemory.GCREF.TO) + + def rawrefcount_next_garbage_pyobj(self): + if self._rrc_gc_list_is_empty(self.rrc_pyobj_garbage_list): + return llmemory.NULL + else: + # rrc_pyobj_garbage_list is not a real list, it just points to + # the first (c_gc_next) and last (c_gc_prev) pyobject in the list + # of live objects that are garbage, so just fix the references + list = self.rrc_pyobj_garbage_list + gchdr = list.c_gc_next + if list.c_gc_prev == gchdr: + list.c_gc_next = list # reached end of list, reset it + else: + list.c_gc_next = gchdr.c_gc_next # move pointer foward + return llmemory.cast_ptr_to_adr(self.rrc_gc_as_pyobj(gchdr)) def rrc_invoke_callback(self): if self.rrc_enabled and (self.rrc_dealloc_pending.non_empty() or - self.rrc_pyobj_isolate_list.c_gc_next <> - self.rrc_pyobj_isolate_list or - self.rrc_pyobj_garbage_list.c_gc_next <> - self.rrc_pyobj_garbage_list): + not self._rrc_gc_list_is_empty( + self.rrc_pyobj_isolate_list) or + not self._rrc_gc_list_is_empty( + self.rrc_pyobj_dead_list) or + not self._rrc_gc_list_is_empty( + self.rrc_pyobj_garbage_list)): self.rrc_dealloc_trigger_callback() def rrc_minor_collection_trace(self): @@ -3392,7 +3399,10 @@ self._rrc_collect_rawrefcount_roots(self.rrc_pyobj_list) # mark all objects reachable from rawrefcounted roots self._rrc_mark_rawrefcount() - self._rrc_mark_garbage() # handle legacy finalizers + self.rrc_state = self.RAWREFCOUNT_STATE_MARKING + if self._rrc_find_garbage(): # handle legacy finalizers + self._rrc_mark_garbage() + self.rrc_state = self.RAWREFCOUNT_STATE_DEFAULT use_cylicrc = not self._rrc_find_finalizer() # modern finalizers else: use_cylicrc = False # don't sweep any objects in cyclic isolates @@ -3450,7 +3460,7 @@ if self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT: if not self._rrc_gc_list_is_empty(self.rrc_pyobj_old_list): self._rrc_gc_list_merge(self.rrc_pyobj_old_list, - self.rrc_pyobj_garbage_list) + self.rrc_pyobj_dead_list) def _rrc_major_free(self, pyobject, surviving_list, surviving_dict): # The pyobject survives if the corresponding obj survives. @@ -3553,11 +3563,65 @@ # now all rawrefcounted objects, which are alive, have a cyclic # refcount > 0 or are marked + def _rrc_find_garbage(self): + found_garbage = False + gchdr = self.rrc_pyobj_old_list.c_gc_next + while gchdr <> self.rrc_pyobj_old_list: + next_old = gchdr.c_gc_next + garbage = self.rrc_finalizer_type(gchdr) == \ + self.RAWREFCOUNT_FINALIZER_LEGACY + if garbage: + self._rrc_move_to_garbage(gchdr) + found_garbage = True + gchdr = next_old + return found_garbage + def _rrc_mark_garbage(self): - self.rrc_state = self.RAWREFCOUNT_STATE_MARKING - # TODO: move all pyobjs to sublist of pyobj_list and reset cyclic refcount - # TODO: mark all pypy-objects with special flag and mark them visited - self.rrc_state = self.RAWREFCOUNT_STATE_DEFAULT + found_garbage = True + # + while found_garbage: + found_garbage = False + gchdr = self.rrc_pyobj_old_list.c_gc_next + while gchdr <> self.rrc_pyobj_old_list: + next_old = gchdr.c_gc_next + alive = (gchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT) > 0 + pyobj = self.rrc_gc_as_pyobj(gchdr) + if pyobj.c_ob_pypy_link <> 0: + intobj = pyobj.c_ob_pypy_link + obj = llmemory.cast_int_to_adr(intobj) + if not alive and self.header(obj).tid & ( + GCFLAG_VISITED | GCFLAG_NO_HEAP_PTRS): + # add fake refcount, to mark it as live + gchdr.c_gc_refs += 1 << self.RAWREFCOUNT_REFS_SHIFT + alive = True + if alive: + self._rrc_move_to_garbage(gchdr) + found_garbage = True + gchdr = next_old + + def _rrc_move_to_garbage(self, gchdr): + pyobj = self.rrc_gc_as_pyobj(gchdr) + # remove from old list + next = gchdr.c_gc_next + next.c_gc_prev = gchdr.c_gc_prev + gchdr.c_gc_prev.c_gc_next = next + # add to beginning of pyobj_list + self._rrc_gc_list_add(self.rrc_pyobj_list, gchdr) + # set as new beginning (and optionally end) of + # pyobj_garbage_list (not a real list, just pointers to + # begin and end) + if self._rrc_gc_list_is_empty(self.rrc_pyobj_garbage_list): + self.rrc_pyobj_garbage_list.c_gc_prev = gchdr + self.rrc_pyobj_garbage_list.c_gc_next = gchdr + # mark referenced objects alive (so objects in the old list + # will be detected as garbage, as they should have a cyclic + # refcount of zero or an unmarked linked pypy object) + self._rrc_traverse(pyobj, 1) + if pyobj.c_ob_pypy_link <> 0: + intobj = pyobj.c_ob_pypy_link + obj = llmemory.cast_int_to_adr(intobj) + self.objects_to_trace.append(obj) + self.visit_all_objects() def _rrc_check_finalizer(self): # Check, if the cyclic isolate from the last collection cycle @@ -3576,7 +3640,7 @@ self.rrc_pyobj_list) else: self._rrc_gc_list_merge(self.rrc_pyobj_old_list, - self.rrc_pyobj_garbage_list) + self.rrc_pyobj_dead_list) def _rrc_find_finalizer(self): found_finalizer = False @@ -3619,6 +3683,11 @@ else: self.rrc_tp_traverse(pyobj, self._rrc_visit_action, None) + def _rrc_gc_list_new(self): + list = lltype.malloc(self.PYOBJ_GC_HDR, flavor='raw', immortal=True) + self._rrc_gc_list_init(list) + return list + def _rrc_gc_list_init(self, pygclist): pygclist.c_gc_next = pygclist pygclist.c_gc_prev = pygclist diff --git a/rpython/memory/gc/test/test_rawrefcount.py b/rpython/memory/gc/test/test_rawrefcount.py --- a/rpython/memory/gc/test/test_rawrefcount.py +++ b/rpython/memory/gc/test/test_rawrefcount.py @@ -51,7 +51,7 @@ if pyobj in self.pyobjs and \ self.pyobj_finalizer.has_key(self.pyobjs.index(pyobj)): # TODO: improve test, so that NONE is returned, if finalizer - # has already been called + # has already been called (only for modern) return self.pyobj_finalizer[self.pyobjs.index(pyobj)] else: return RAWREFCOUNT_FINALIZER_NONE @@ -472,7 +472,7 @@ rooted = attr['rooted'] == "y" if 'rooted' in attr else False ext_refcnt = int(attr['ext_refcnt']) if 'ext_refcnt' in attr else 0 finalizer = attr['finalizer'] if 'finalizer' in attr else None - if finalizer <> None: + if finalizer == "modern": finalizers = True resurrect = attr['resurrect'] if 'resurrect' in attr else None delete = attr['delete'] if 'delete' in attr else None @@ -632,12 +632,10 @@ while next_garbage <> lltype.nullptr(llmemory.GCREF.TO): garbage_pypy.append(next_garbage) next_garbage = self.gc.rawrefcount_next_garbage_pypy() - last_pyobj = llmemory.NULL - next = self.gc.rawrefcount_next_garbage_pyobj(last_pyobj) + next = self.gc.rawrefcount_next_garbage_pyobj() while next <> llmemory.NULL: garbage_pyobj.append(next) - next = self.gc.rawrefcount_next_garbage_pyobj(last_pyobj) - last_pyobj = next + next = self.gc.rawrefcount_next_garbage_pyobj() self.gc.rawrefcount_end_garbage() # do a collection to find cyclic isolates and clean them, if there are diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -528,8 +528,8 @@ GCClass.rawrefcount_next_garbage_pypy, [s_gc], s_gcref, inline = True) self.rawrefcount_next_garbage_pyobj_ptr = getfn( - GCClass.rawrefcount_next_garbage_pyobj, [s_gc, SomeAddress()], - SomeAddress(), inline = True) + GCClass.rawrefcount_next_garbage_pyobj, [s_gc], SomeAddress(), + inline = True) if GCClass.can_usually_pin_objects: self.pin_ptr = getfn(GCClass.pin, @@ -1453,12 +1453,9 @@ resultvar=hop.spaceop.result) def gct_gc_rawrefcount_next_garbage_pyobj(self, hop): - [v_pyobject] = hop.spaceop.args - assert v_pyobject.concretetype == llmemory.Address assert hop.spaceop.result.concretetype == llmemory.Address hop.genop("direct_call", - [self.rawrefcount_next_garbage_pyobj_ptr, self.c_const_gc, - v_pyobject], + [self.rawrefcount_next_garbage_pyobj_ptr, self.c_const_gc], resultvar=hop.spaceop.result) def _set_into_gc_array_part(self, op): diff --git a/rpython/rlib/rawrefcount.py b/rpython/rlib/rawrefcount.py --- a/rpython/rlib/rawrefcount.py +++ b/rpython/rlib/rawrefcount.py @@ -158,7 +158,7 @@ return None @not_rpython -def next_garbage_pyobj(OB_PTR_TYPE, curr_pyobj): +def next_garbage_pyobj(OB_PTR_TYPE): return lltype.nullptr(OB_PTR_TYPE.TO) @not_rpython @@ -383,7 +383,8 @@ return _spec_p(hop, v_p) class Entry(ExtRegistryEntry): - _about_ = (next_dead, cyclic_garbage_head, next_cyclic_isolate) + _about_ = (next_dead, cyclic_garbage_head, next_cyclic_isolate, + next_garbage_pyobj) def compute_result_annotation(self, s_OB_PTR_TYPE): from rpython.rtyper.llannotation import lltype_to_annotation @@ -397,28 +398,13 @@ name = 'gc_rawrefcount_cyclic_garbage_head' elif self.instance is next_cyclic_isolate: name = 'gc_rawrefcount_next_cyclic_isolate' + elif self.instance is next_garbage_pyobj: + name = 'gc_rawrefcount_next_garbage_pyobj' hop.exception_cannot_occur() v_ob = hop.genop(name, [], resulttype = llmemory.Address) return _spec_ob(hop, v_ob) class Entry(ExtRegistryEntry): - _about_ = next_garbage_pyobj - - def compute_result_annotation(self, s_OB_PTR_TYPE, s_ob): - from rpython.rtyper.llannotation import lltype_to_annotation, SomePtr - assert s_OB_PTR_TYPE.is_constant() - assert isinstance(s_ob, SomePtr) - return lltype_to_annotation(s_OB_PTR_TYPE.const) - - def specialize_call(self, hop): - hop.exception_cannot_occur() - v_ob = hop.inputarg(hop.args_r[1], arg=1) - v_ob_res = hop.genop('gc_rawrefcount_next_garbage_pyobj', - [_unspec_ob(hop, v_ob)], - resulttype = llmemory.Address) - return _spec_ob(hop, v_ob_res) - -class Entry(ExtRegistryEntry): _about_ = next_garbage_pypy def compute_result_annotation(self, s_Class): From pypy.commits at gmail.com Mon Mar 4 08:11:21 2019 From: pypy.commits at gmail.com (mattip) Date: Mon, 04 Mar 2019 05:11:21 -0800 (PST) Subject: [pypy-commit] pypy newmemoryview-app-level: handle memoryview of basic ctype arrays Pointers and Structures Message-ID: <5c7d23f9.1c69fb81.66442.a67c@mx.google.com> Author: Matti Picus Branch: newmemoryview-app-level Changeset: r96207:1c89b5e95a50 Date: 2019-03-04 15:07 +0200 http://bitbucket.org/pypy/pypy/changeset/1c89b5e95a50/ Log: handle memoryview of basic ctype arrays Pointers and Structures diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py --- a/lib_pypy/_ctypes/array.py +++ b/lib_pypy/_ctypes/array.py @@ -4,7 +4,7 @@ from _ctypes.basics import _CData, cdata_from_address, _CDataMeta, sizeof from _ctypes.basics import keepalive_key, store_reference, ensure_objects from _ctypes.basics import CArgObject, as_ffi_pointer -import sys, __pypy__ +import sys, __pypy__, struct class ArrayMeta(_CDataMeta): def __new__(self, name, cls, typedict): @@ -252,9 +252,12 @@ except AttributeError: break obj = obj[0] - + fmt = get_format_str(obj._type_) - itemsize = len(memoryview(obj[0])) + try: + itemsize = struct.calcsize(fmt[1:]) + except: + itemsize = len(memoryview(obj[0])) return __pypy__.newmemoryview(memoryview(self._buffer), itemsize, fmt, shape) ARRAY_CACHE = {} @@ -288,7 +291,8 @@ bo = byteorder[sys.byteorder] flds = [] for name, obj in typ._fields_: - ch = get_format_str(obj) + # Trim off the leading '<' or '>' + ch = get_format_str(obj)[1:] if (ch) == 'B': flds.append(byteorder[sys.byteorder]) else: @@ -299,6 +303,7 @@ flds.append(':') return 'T{' + ''.join(flds) + '}' elif hasattr(typ, '_type_'): - return typ._type_ + ch = typ._type_ + return byteorder[sys.byteorder] + ch else: raise ValueError('cannot get format string for %r' % typ) diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py --- a/lib_pypy/_ctypes/pointer.py +++ b/lib_pypy/_ctypes/pointer.py @@ -7,8 +7,7 @@ from _ctypes.array import Array, array_get_slice_params, array_slice_getitem,\ array_slice_setitem -try: from __pypy__ import builtinify -except ImportError: builtinify = lambda f: f +from __pypy__ import builtinify, newmemoryview # This cache maps types to pointers to them. _pointer_type_cache = {} @@ -135,6 +134,9 @@ def _as_ffi_pointer_(self, ffitype): return as_ffi_pointer(self, ffitype) + def __buffer__(self, flags): + mv = memoryview(self.getcontents()) + return newmemoryview(mv, mv.itemsize, '&' + mv.format, mv.shape) def _cast_addr(obj, _, tp): if not (isinstance(tp, _CDataMeta) and tp._is_pointer_like()): diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -2,9 +2,9 @@ import _rawffi from _ctypes.basics import _CData, _CDataMeta, keepalive_key,\ store_reference, ensure_objects, CArgObject -from _ctypes.array import Array +from _ctypes.array import Array, get_format_str from _ctypes.pointer import _Pointer -import inspect +import inspect, __pypy__ def names_and_fields(self, _fields_, superclass, anonymous_fields=None): @@ -299,6 +299,10 @@ def _to_ffi_param(self): return self._buffer + def __buffer__(self, flags): + fmt = get_format_str(self) + itemsize = type(self)._sizeofinstances() + return __pypy__.newmemoryview(memoryview(self._buffer), itemsize, fmt) class StructureMeta(StructOrUnionMeta): _is_union = False From pypy.commits at gmail.com Mon Mar 4 08:11:23 2019 From: pypy.commits at gmail.com (mattip) Date: Mon, 04 Mar 2019 05:11:23 -0800 (PST) Subject: [pypy-commit] pypy newmemoryview-app-level: hack to expand the max_ndims past numpy's limit Message-ID: <5c7d23fb.1c69fb81.481bd.2d82@mx.google.com> Author: Matti Picus Branch: newmemoryview-app-level Changeset: r96208:eb2e008d2181 Date: 2019-03-04 15:09 +0200 http://bitbucket.org/pypy/pypy/changeset/eb2e008d2181/ Log: hack to expand the max_ndims past numpy's limit diff --git a/pypy/module/cpyext/parse/cpyext_memoryobject.h b/pypy/module/cpyext/parse/cpyext_memoryobject.h --- a/pypy/module/cpyext/parse/cpyext_memoryobject.h +++ b/pypy/module/cpyext/parse/cpyext_memoryobject.h @@ -1,6 +1,12 @@ /* The struct is declared here but it shouldn't be considered public. Don't access those fields directly, use the functions instead! */ + + +/* this is wrong, PyMemoryViewObject should use PyObject_VAR_HEAD, and use + ob_data[1] to hold the shapes, strides, and offsets for the view. Then + we should use specialized allocators (that break the cpyext model) to + allocate ob_data = malloc(sizeof(Py_ssize_t) * view.ndims * 3) */ typedef struct { PyObject_HEAD Py_buffer view; diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h --- a/pypy/module/cpyext/parse/cpyext_object.h +++ b/pypy/module/cpyext/parse/cpyext_object.h @@ -68,7 +68,8 @@ typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **); /* Py3k buffer interface, adapted for PyPy */ -#define Py_MAX_NDIMS 32 +/* XXX remove this constant, us a PyObject_VAR_HEAD instead */ +#define Py_MAX_NDIMS 36 #define Py_MAX_FMT 128 typedef struct bufferinfo { void *buf; From pypy.commits at gmail.com Mon Mar 4 09:11:46 2019 From: pypy.commits at gmail.com (mattip) Date: Mon, 04 Mar 2019 06:11:46 -0800 (PST) Subject: [pypy-commit] buildbot default: add notes from bencher4 how to run/setup chroot Message-ID: <5c7d3222.1c69fb81.df103.da5f@mx.google.com> Author: Matti Picus Branch: Changeset: r1070:d48bd03cc863 Date: 2019-03-04 16:11 +0200 http://bitbucket.org/pypy/buildbot/changeset/d48bd03cc863/ Log: add notes from bencher4 how to run/setup chroot diff --git a/README-CHROOT b/README-CHROOT new file mode 100644 --- /dev/null +++ b/README-CHROOT @@ -0,0 +1,55 @@ +Copied from bencher4, the venerable linux64 builder +--------------------------------------------------------- +Buildslave runs in a chroot: + + LANG=C chroot /mnt/ubuntu /bin/bash + su - buildslave + + +Commands to run once after a reboot to set the chroot: + + >>> have been copied into /etc/rc.local + +mount --bind /tmp /home/tmp +mount --rbind /extra1 /home/extra1 +mount --rbind /proc /mnt/ubuntu/proc +mount --rbind /sys /mnt/ubuntu/sys +mount --rbind /dev /mnt/ubuntu/dev +mount --rbind /run/lock /mnt/ubuntu/run/lock +mount --rbind /run/shm /mnt/ubuntu/run/shm + + +--------------------------------------------------------- +And here is the same for the 32-bit chroot +--------------------------------------------------------- +32 bit buildslave runs in a debian strech chroot, +chosen since debian comes with gcc6: + + LANG=C chroot /extra1/stretch32 /bin/bash + su - buildslave + +Commands to run once after a reboot to set the chroot: + + >>> have been copied into /etc/rc.local +mount --bind /tmp /extra1/stretch32/tmp +mount --rbind /proc /extra1/stretch32/proc +mount --rbind /sys /extra1/stretch32/sys +mount --rbind /dev /extra1/stretch32/dev +mount --rbind /run/lock /extra1/stretch32/run/lock + + +Commands to build the chroot + +mkdir /extra1/stretch32 +sudo debootstrap --variant=buildd --arch=i386 stretch \ + /extra1/stretch32 http://deb.debian.org/debian +rm /extra1/strech32/tmp +LANG=C chroot /extra1/stretch32 +echo stretch32 > /etc/debian_chroot +adduser buildslave +# go back out out of the chroot and set buildslave's UID to the UID on the host +chmod buildslave.buildslave -R /extra1/stretch32/home/buildslave +apt install virtualenv buildbot-slave python-pytest python-hypothesis + +and more from the instructions on https://bitbucket.org/pypy/buildbot + From pypy.commits at gmail.com Mon Mar 4 15:33:55 2019 From: pypy.commits at gmail.com (mattip) Date: Mon, 04 Mar 2019 12:33:55 -0800 (PST) Subject: [pypy-commit] buildbot default: update README-CHROOT Message-ID: <5c7d8bb3.1c69fb81.6b11.e2d1@mx.google.com> Author: Matti Picus Branch: Changeset: r1071:817276678139 Date: 2019-03-04 22:28 +0200 http://bitbucket.org/pypy/buildbot/changeset/817276678139/ Log: update README-CHROOT diff --git a/README-CHROOT b/README-CHROOT --- a/README-CHROOT +++ b/README-CHROOT @@ -31,7 +31,7 @@ Commands to run once after a reboot to set the chroot: >>> have been copied into /etc/rc.local -mount --bind /tmp /extra1/stretch32/tmp +mount --rbind /tmp /extra1/stretch32/tmp mount --rbind /proc /extra1/stretch32/proc mount --rbind /sys /extra1/stretch32/sys mount --rbind /dev /extra1/stretch32/dev @@ -43,7 +43,6 @@ mkdir /extra1/stretch32 sudo debootstrap --variant=buildd --arch=i386 stretch \ /extra1/stretch32 http://deb.debian.org/debian -rm /extra1/strech32/tmp LANG=C chroot /extra1/stretch32 echo stretch32 > /etc/debian_chroot adduser buildslave From pypy.commits at gmail.com Mon Mar 4 15:33:58 2019 From: pypy.commits at gmail.com (mattip) Date: Mon, 04 Mar 2019 12:33:58 -0800 (PST) Subject: [pypy-commit] buildbot default: add benchmarker64 slave, remove raspberry-pi cross builders Message-ID: <5c7d8bb6.1c69fb81.53b05.adc3@mx.google.com> Author: Matti Picus Branch: Changeset: r1072:58d0b93576d2 Date: 2019-03-04 22:28 +0200 http://bitbucket.org/pypy/buildbot/changeset/58d0b93576d2/ Log: add benchmarker64 slave, remove raspberry-pi cross builders diff --git a/bot2/pypybuildbot/arm_master.py b/bot2/pypybuildbot/arm_master.py --- a/bot2/pypybuildbot/arm_master.py +++ b/bot2/pypybuildbot/arm_master.py @@ -87,32 +87,32 @@ # LINUXARMHFOWN = "own-linux-armhf" LINUXARMHFRPYTHON = "rpython-linux-armhf" -APPLVLLINUXARM = "pypy-c-app-level-linux-armel" +# APPLVLLINUXARM = "pypy-c-app-level-linux-armel" APPLVLLINUXARMHF_v7 = "pypy-c-app-level-linux-armhf-v7" -APPLVLLINUXARMHF_RASPBIAN = "pypy-c-app-level-linux-armhf-raspbian" +# APPLVLLINUXARMHF_RASPBIAN = "pypy-c-app-level-linux-armhf-raspbian" -JITLINUXARM = "pypy-c-jit-linux-armel" +# JITLINUXARM = "pypy-c-jit-linux-armel" JITLINUXARMHF_v7 = "pypy-c-jit-linux-armhf-v7" JITLINUXARMHF_RASPBIAN = "pypy-c-jit-linux-armhf-raspbian" -JITBACKENDONLYLINUXARMEL = "jitbackendonly-own-linux-armel" +# JITBACKENDONLYLINUXARMEL = "jitbackendonly-own-linux-armel" JITBACKENDONLYLINUXARMHF = "jitbackendonly-own-linux-armhf" JITBACKENDONLYLINUXARMHF_v7 = "jitbackendonly-own-linux-armhf-v7" # build only BUILDLINUXARM = "build-pypy-c-linux-armel" -BUILDJITLINUXARM = "build-pypy-c-jit-linux-armel" +# BUILDJITLINUXARM = "build-pypy-c-jit-linux-armel" BUILDLINUXARMHF_RASPBIAN = "build-pypy-c-linux-armhf-raspbian" BUILDJITLINUXARMHF_RASPBIAN = "build-pypy-c-jit-linux-armhf-raspbian" builderNames = [ - APPLVLLINUXARM, + # APPLVLLINUXARM, APPLVLLINUXARMHF_v7, - APPLVLLINUXARMHF_RASPBIAN, - JITLINUXARM, + # APPLVLLINUXARMHF_RASPBIAN, + # JITLINUXARM, JITLINUXARMHF_v7, JITLINUXARMHF_RASPBIAN, - JITBACKENDONLYLINUXARMEL, + # JITBACKENDONLYLINUXARMEL, JITBACKENDONLYLINUXARMHF, JITBACKENDONLYLINUXARMHF_v7, #BUILDLINUXARM, @@ -141,7 +141,7 @@ ), Nightly("nightly-arm-0-01", [ - JITBACKENDONLYLINUXARMEL, # on hhu-imx.53 + # JITBACKENDONLYLINUXARMEL, # on hhu-imx.53 JITBACKENDONLYLINUXARMHF, JITBACKENDONLYLINUXARMHF_v7, # on cubieboard-bob ], branch='default', hour=0, minute=0, onlyIfChanged=True, @@ -149,15 +149,15 @@ change_filter=filter.ChangeFilter(branch='default'), ), - Triggerable("APPLVLLINUXARM_scheduler", [ - APPLVLLINUXARM, # triggered by BUILDLINUXARM, on hhu-beagleboard - ]), + #Triggerable("APPLVLLINUXARM_scheduler", [ + # APPLVLLINUXARM, # triggered by BUILDLINUXARM, on hhu-beagleboard + #]), - Triggerable("JITLINUXARM_scheduler", [ - JITLINUXARM, # triggered by BUILDJITLINUXARM, on hhu-beagleboard - ]), + #Triggerable("JITLINUXARM_scheduler", [ + # JITLINUXARM, # triggered by BUILDJITLINUXARM, on hhu-beagleboard + #]), Triggerable("APPLVLLINUXARMHF_RASPBIAN_scheduler", [ - APPLVLLINUXARMHF_RASPBIAN, # triggered by BUILDLINUXARMHF_RASPBIAN + # APPLVLLINUXARMHF_RASPBIAN, # triggered by BUILDLINUXARMHF_RASPBIAN APPLVLLINUXARMHF_v7, # triggered by BUILDLINUXARMHF_RASPBIAN, on cubieboard-bob ]), @@ -172,13 +172,13 @@ # ARM # armel ## armv7 - {"name": JITBACKENDONLYLINUXARMEL, - "slavenames": ['hhu-i.mx53'], - "builddir": JITBACKENDONLYLINUXARMEL, - "factory": pypyJitBackendOnlyRPythonTestFactoryARM, - "category": 'linux-armel', - "locks": [ARMBoardLock.access('counting')], - }, + #{"name": JITBACKENDONLYLINUXARMEL, + # "slavenames": ['hhu-i.mx53'], + # "builddir": JITBACKENDONLYLINUXARMEL, + # "factory": pypyJitBackendOnlyRPythonTestFactoryARM, + # "category": 'linux-armel', + # "locks": [ARMBoardLock.access('counting')], + # }, # armhf ## armv6 {"name": JITBACKENDONLYLINUXARMHF, @@ -198,28 +198,28 @@ }, # app level builders ## armv7 softfloat - {"name": APPLVLLINUXARM, - "slavenames": ["hhu-beagleboard"], - "builddir": APPLVLLINUXARM, - "factory": pypyARMTranslatedAppLevelTestFactory, - "category": "linux-armel", - "locks": [ARMBoardLock.access('counting')], - }, - {"name": JITLINUXARM, - "slavenames": ["hhu-beagleboard"], - 'builddir': JITLINUXARM, - 'factory': pypyARMJITTranslatedTestFactory, - 'category': 'linux-armel', - "locks": [ARMBoardLock.access('counting')], - }, + #{"name": APPLVLLINUXARM, + # "slavenames": ["hhu-beagleboard"], + # "builddir": APPLVLLINUXARM, + # "factory": pypyARMTranslatedAppLevelTestFactory, + # "category": "linux-armel", + # "locks": [ARMBoardLock.access('counting')], + # }, + #{"name": JITLINUXARM, + # "slavenames": ["hhu-beagleboard"], + # 'builddir': JITLINUXARM, + # 'factory': pypyARMJITTranslatedTestFactory, + # 'category': 'linux-armel', + # "locks": [ARMBoardLock.access('counting')], + # }, ## armv6 hardfloat - {"name": APPLVLLINUXARMHF_RASPBIAN, - "slavenames": ['hhu-raspberry-pi', 'hhu-pypy-pi', 'hhu-pypy-pi2'], - "builddir": APPLVLLINUXARMHF_RASPBIAN, - "factory": pypyARMHF_RASPBIAN_TranslatedAppLevelTestFactory, - "category": "linux-armhf", - "locks": [ARMBoardLock.access('counting')], - }, + #{"name": APPLVLLINUXARMHF_RASPBIAN, + # "slavenames": ['hhu-raspberry-pi', 'hhu-pypy-pi', 'hhu-pypy-pi2'], + # "builddir": APPLVLLINUXARMHF_RASPBIAN, + # "factory": pypyARMHF_RASPBIAN_TranslatedAppLevelTestFactory, + # "category": "linux-armhf", + # "locks": [ARMBoardLock.access('counting')], + # }, {"name": JITLINUXARMHF_RASPBIAN, "slavenames": ['hhu-raspberry-pi', 'hhu-pypy-pi', 'hhu-pypy-pi2'], 'builddir': JITLINUXARMHF_RASPBIAN, diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -433,7 +433,7 @@ }, {"name": LINUX64OWN, #"slavenames": ["bencher4", "speed-old"], - "slavenames": ["bencher4"], + "slavenames": ["bencher4", "benchmarker64"], "builddir": LINUX64OWN, "factory": pypyOwnTestFactory, "category": 'linux64', @@ -441,7 +441,7 @@ }, {"name": LINUX64RPYTHON, #"slavenames": ["bencher4", "speed-old"], - "slavenames": ["bencher4"], + "slavenames": ["bencher4", "benchmarker64"], "builddir": LINUX64RPYTHON, "factory": pypyRPythonTestFactory, "category": 'linux64', @@ -489,7 +489,7 @@ }, {'name': JITLINUX64, #'slavenames': ["bencher4", "speed-old"], - 'slavenames': ["bencher4"], + 'slavenames': ["bencher4", "benchmarker64"], 'builddir': JITLINUX64, 'factory': pypyJITTranslatedTestFactory64, 'category': 'linux64', @@ -576,7 +576,7 @@ "category": 'linux-ppc64', }, {'name': NUMPY_64, - 'slavenames': ["bencher4"], + 'slavenames': ["bencher4", "benchmarker64"], 'builddir': NUMPY_64, 'factory': pypyNumpyCompatability, 'category': 'numpy', From pypy.commits at gmail.com Tue Mar 5 04:10:30 2019 From: pypy.commits at gmail.com (arigo) Date: Tue, 05 Mar 2019 01:10:30 -0800 (PST) Subject: [pypy-commit] cffi default: Document issue 391 Message-ID: <5c7e3d06.1c69fb81.418a5.0883@mx.google.com> Author: Armin Rigo Branch: Changeset: r3239:6d435908617d Date: 2019-03-05 10:11 +0100 http://bitbucket.org/cffi/cffi/changeset/6d435908617d/ Log: Document issue 391 diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -6019,8 +6019,8 @@ PyErr_SetString(PyExc_MemoryError, "Cannot allocate write+execute memory for ffi.callback(). " "You might be running on a system that prevents this. " - "For more information, see https://cffi.readthedocs.io/en/latest" - "/using.html#callbacks-old-style"); + "For more information, see " + "https://cffi.readthedocs.io/en/latest/using.html#callbacks"); return NULL; } cd = PyObject_GC_New(CDataObject_closure, &CDataOwningGC_Type); diff --git a/doc/source/using.rst b/doc/source/using.rst --- a/doc/source/using.rst +++ b/doc/source/using.rst @@ -876,11 +876,27 @@ protections can interfere (for example, on SELinux you need to run with ``deny_execmem`` set to ``off``). - Note also that a cffi fix for the latter issue was attempted---see + - `On Mac OS X,`__ you need to give your application the entitlement + ``com.apple.security.cs.allow-unsigned-executable-memory``. + + Note also that a cffi fix for this issue was attempted---see the ``ffi_closure_alloc`` branch---but was not merged because it creates potential `memory corruption`__ with ``fork()``. + In other words: yes, it is dangerous to allow write+execute memory in your + program; that's why the various "hardening" options above exist. But at + the same time, these options open wide the door to another attack: if the + program forks and then attempts to call any of the ``ffi.callback()``, then + this immediately results in a crash---or, with a minimal amount of work + from an attacker, arbitrary code execution. To me it sounds even more + dangerous than the original problem, and that's why cffi is not playing + along. + + To fix the issue once and for all on the affected platforms, you need + to refactor the involved code so that it no longer uses ``ffi.callback()``. + .. __: https://github.com/pyca/pyopenssl/issues/596 +.. __: https://bitbucket.org/cffi/cffi/issues/391/ .. __: https://bugzilla.redhat.com/show_bug.cgi?id=1249685 Warning: like ffi.new(), ffi.callback() returns a cdata that has From pypy.commits at gmail.com Tue Mar 5 06:23:38 2019 From: pypy.commits at gmail.com (mattip) Date: Tue, 05 Mar 2019 03:23:38 -0800 (PST) Subject: [pypy-commit] buildbot default: document more required debian packages Message-ID: <5c7e5c3a.1c69fb81.134f7.6e70@mx.google.com> Author: Matti Picus Branch: Changeset: r1073:103d5aa76305 Date: 2019-03-05 13:23 +0200 http://bitbucket.org/pypy/buildbot/changeset/103d5aa76305/ Log: document more required debian packages diff --git a/README-CHROOT b/README-CHROOT --- a/README-CHROOT +++ b/README-CHROOT @@ -10,7 +10,7 @@ >>> have been copied into /etc/rc.local -mount --bind /tmp /home/tmp +mount --rbind /tmp /home/tmp mount --rbind /extra1 /home/extra1 mount --rbind /proc /mnt/ubuntu/proc mount --rbind /sys /mnt/ubuntu/sys @@ -48,7 +48,9 @@ adduser buildslave # go back out out of the chroot and set buildslave's UID to the UID on the host chmod buildslave.buildslave -R /extra1/stretch32/home/buildslave -apt install virtualenv buildbot-slave python-pytest python-hypothesis + +apt install virtualenv buildbot-slave python-pytest python-hypothesis \ +netbase gdb ncurses-term pypy and more from the instructions on https://bitbucket.org/pypy/buildbot From pypy.commits at gmail.com Tue Mar 5 06:58:44 2019 From: pypy.commits at gmail.com (mattip) Date: Tue, 05 Mar 2019 03:58:44 -0800 (PST) Subject: [pypy-commit] pypy newmemoryview-app-level: fix failing tests, document branch and new app-level functionality Message-ID: <5c7e6474.1c69fb81.33c49.6598@mx.google.com> Author: Matti Picus Branch: newmemoryview-app-level Changeset: r96209:e7f2a1847440 Date: 2019-03-05 13:58 +0200 http://bitbucket.org/pypy/pypy/changeset/e7f2a1847440/ Log: fix failing tests, document branch and new app-level functionality diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py --- a/lib_pypy/_ctypes/array.py +++ b/lib_pypy/_ctypes/array.py @@ -249,7 +249,7 @@ shape.append(obj._length_) try: obj[0]._length_ - except AttributeError: + except (AttributeError, IndexError): break obj = obj[0] @@ -257,7 +257,7 @@ try: itemsize = struct.calcsize(fmt[1:]) except: - itemsize = len(memoryview(obj[0])) + itemsize = len(buffer(obj[0])) return __pypy__.newmemoryview(memoryview(self._buffer), itemsize, fmt, shape) ARRAY_CACHE = {} diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst --- a/pypy/doc/__pypy__-module.rst +++ b/pypy/doc/__pypy__-module.rst @@ -18,8 +18,17 @@ - ``bytebuffer(length)``: return a new read-write buffer of the given length. It works like a simplified array of characters (actually, depending on the configuration the ``array`` module internally uses this). + - ``attach_gdb()``: start a GDB at the interpreter-level (or a PDB before translation). + - ``newmemoryview(buffer, itemsize, format, shape=None, strides=None)``: + create a `memoryview` instance with the data from ``buffer`` and the + specified itemsize, format, and optional shape and strides. + + - ``bufferable``: a base class that must override the + ``__buffer__(self, flags)`` method. This method should return a memoryview + instance of the class instance. It is called by the C-API's ``tp_as_buffer. + bf_getbuffer``. Transparent Proxy Functionality ------------------------------- diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -37,3 +37,9 @@ Use utf8 internally to represent unicode, with the goal of never using rpython-level unicode +.. branch: newmemoryview + +Since _ctypes is implemented in pure python over libffi, add interfaces and +methods to support the buffer interface from python. Specifically, add a +``__pypy__.newmemoryview`` function to create a memoryview and extend the use +of the PyPy-specific ``__buffer__`` class method. From pypy.commits at gmail.com Tue Mar 5 12:51:11 2019 From: pypy.commits at gmail.com (mattip) Date: Tue, 05 Mar 2019 09:51:11 -0800 (PST) Subject: [pypy-commit] pypy newmemoryview-app-level: finx branch name Message-ID: <5c7eb70f.1c69fb81.fa5cb.4879@mx.google.com> Author: Matti Picus Branch: newmemoryview-app-level Changeset: r96210:e1de22f3f071 Date: 2019-03-05 18:27 +0200 http://bitbucket.org/pypy/pypy/changeset/e1de22f3f071/ Log: finx branch name diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -37,7 +37,7 @@ Use utf8 internally to represent unicode, with the goal of never using rpython-level unicode -.. branch: newmemoryview +.. branch: newmemoryview-app-level Since _ctypes is implemented in pure python over libffi, add interfaces and methods to support the buffer interface from python. Specifically, add a From pypy.commits at gmail.com Tue Mar 5 12:51:13 2019 From: pypy.commits at gmail.com (mattip) Date: Tue, 05 Mar 2019 09:51:13 -0800 (PST) Subject: [pypy-commit] pypy default: Expand documentation of __pypy_ module. Corrections, additions, removals welcome Message-ID: <5c7eb711.1c69fb81.a2ad7.074b@mx.google.com> Author: Matti Picus Branch: Changeset: r96211:c8c8b63926be Date: 2019-03-05 19:50 +0200 http://bitbucket.org/pypy/pypy/changeset/c8c8b63926be/ Log: Expand documentation of __pypy_ module. Corrections, additions, removals welcome diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst --- a/pypy/doc/__pypy__-module.rst +++ b/pypy/doc/__pypy__-module.rst @@ -1,25 +1,139 @@ -.. comment: this document is very incomplete, should we generate - it automatically? +.. comment: this document may get out of synch with the code, but to generate + it automatically we would need to use pypy to run sphinx-build The ``__pypy__`` module ======================= The ``__pypy__`` module is the main entry point to special features provided -by PyPy's standard interpreter. Its content depends on :doc:`configuration options ` -which may add new functionality and functions whose existence or non-existence -indicates the presence of such features. +by PyPy's standard interpreter. Its content depends on :doc:`configuration +options ` which may add new functionality and functions whose +existence or non-existence indicates the presence of such features. These are +generally used for compatibility when writing pure python modules that in +CPython are written in C. Not available in CPython, and so must be used inside a +``if platform.python_implementation == 'PyPy'`` block or otherwise hidden from +the CPython interpreter. Generally available functionality --------------------------------- - - ``internal_repr(obj)``: return the interpreter-level representation of an - object. - - ``bytebuffer(length)``: return a new read-write buffer of the given length. - It works like a simplified array of characters (actually, depending on the - configuration the ``array`` module internally uses this). - - ``attach_gdb()``: start a GDB at the interpreter-level (or a PDB before translation). + - ``internal_repr(obj)``: return the interpreter-level representation of an + object. + - ``bytebuffer(length)``: return a new read-write buffer of the given length. + It works like a simplified array of characters (actually, depending on the + configuration the ``array`` module internally uses this). + - ``attach_gdb()``: start a GDB at the interpreter-level (or a PDB before translation). + + - ``identity_dict(object)``: A dictionary that considers keys by object identity. + Distinct objects will have separate entries even if they compare equal. + All objects can be used as keys, even non-hashable ones --- but avoid using + immutable objects like integers: two int objects 42 may or may not be + internally the same object. + + - ``set_debug`` + - ``debug_start`` + - ``debug_stop`` + - ``debug_print`` + - ``debug_print_once`` + - ``debug_flush`` + - ``debug_read_timestamp`` + - ``debug_get_timestamp_unit`` + + - ``builtinify(func)``: To implement at app-level modules that are, in CPython, + implemented in C: this decorator protects a function from being ever bound + like a method. Useful because some tests do things like put a "built-in" + function on a class and access it via the instance. + + - ``hidden_applevel(func)``: Decorator that hides a function's frame from + app-level + + - ``get_hidden_tb()``: Return the traceback of the current exception being + handled by a frame hidden from applevel. + + - ``lookup_special(obj, meth)``: Lookup up a special method on an object. + - ``do_what_I_mean`` + + - ``resizelist_hint(...)``: Reallocate the underlying storage of the argument + list to sizehint + + - ``newlist_hint(...)``: Create a new empty list that has an underlying + storage of length sizehint + + - ``add_memory_pressure(bytes)``: Add memory pressure of estimate bytes. + Useful when calling a C function that internally allocates a big chunk of + memory. This instructs the GC to garbage collect sooner than it would + otherwise. + + - ``newdict(type)``: Create a normal dict with a special implementation + strategy. ``type`` is a string and can be: + + * ``"module"`` - equivalent to ``some_module.__dict__`` + + * ``"instance"`` - equivalent to an instance dict with a not-changing-much + set of keys + + * ``"kwargs"`` - keyword args dict equivalent of what you get from + ``**kwargs`` in a function, optimized for passing around + + * ``"strdict"`` - string-key only dict. This one should be chosen + automatically + + - ``reversed_dict``: Enumerate the keys in a dictionary object in reversed + order. This is a ``__pypy__`` function instead of being simply done by + calling reversed(), for CPython compatibility: dictionaries are ordered in + PyPY but not in Cpython2.7. You should use the collections.OrderedDict + class for cases where ordering is important. That class implements + ``__reversed__`` by calling __pypy__.reversed_dict() + + - ``dict_popitem_first``: Interp-level implementation of + ``OrderedDict.popitem(last=False)``. + + - ``delitem_if_value_is`` Atomic equivalent to: ``if dict.get(key) is value: + del dict[key]``. + + SPECIAL USE CASES ONLY! Avoid using on dicts which are specialized, + e.g. to ``int`` or ``str`` keys, because it switches to the object + strategy. Also, the ``is`` operation is really pointer equality, so avoid + using it if ``value`` is an immutable object like ``int`` or ``str``. + + - ``move_to_end``: Move the key in a dictionary object into the first or last + position. This is used in Python 3.x to implement ``OrderedDict.move_to_end()``. + + - ``strategy(dict or list or set)``: Return the underlying strategy currently + used by the object + + - ``specialized_zip_2_lists`` + - ``locals_to_fast`` + - ``set_code_callback`` + - ``save_module_content_for_future_reload`` + - ``decode_long`` + - ``side_effects_ok``: For use with the reverse-debugger: this function + normally returns True, but will return False if we are evaluating a + debugging command like a watchpoint. You are responsible for not doing any + side effect at all (including no caching) when evaluating watchpoints. This + function is meant to help a bit---you can write:: + + if not __pypy__.side_effects_ok(): + skip the caching logic + + inside getter methods or properties, to make them usable from + watchpoints. Note that you need to re-run ``REVDB=.. pypy`` + after changing the Python code. + + - ``stack_almost_full``: Return True if the stack is more than 15/16th full. + - ``pyos_inputhook``: Call PyOS_InputHook() from the CPython C API + - ``os.real_getenv(...)`` gets OS environment variables skipping python code + - ``_pypydatetime`` provides base classes with correct C API interactions for + the pure-python ``datetime`` stdlib module + +Fast String Concatenation +------------------------- +Rather than in-place concatenation ``+=``, use these to enable fast, minimal +copy, string building. + + - ``builders.StringBuilder`` + - ``builders.UnicodeBuilder`` Transparent Proxy Functionality ------------------------------- @@ -34,6 +148,30 @@ its controller. Otherwise return None. +Additional Clocks for Timing +---------------------------- +The ``time`` submodule exposes the platform-dependent clock types such as +``CLOCK_BOOTTIME``, ``CLOCK_MONOTONIC``, ``CLOCK_MONOTONIC_COARSE``, +``CLOCK_MONOTONIC_RAW`` and two functions: + + - ``clock_gettime(m)`` which returns the clock type time in seconds and + - ``clock_getres(m)`` which returns the clock resolution in seconds. + +Extended Signal Handling +------------------------ +``thread.signals_enbaled`` is a context manager to use in non-main threads. + enables receiving signals in a "with" statement. More precisely, if a + signal is received by the process, then the signal handler might be + called either in the main thread (as usual) or within another thread + that is within a "with signals_enabled:". This other thread should be + ready to handle unexpected exceptions that the signal handler might + raise --- notably KeyboardInterrupt. + +Integer Operations with Overflow +-------------------------------- + - ``intop`` provides a module with integer operations that have + two-complement overflow behaviour instead of overflowing to longs + Functionality available on py.py (not after translation) -------------------------------------------------------- From pypy.commits at gmail.com Tue Mar 5 14:26:37 2019 From: pypy.commits at gmail.com (mattip) Date: Tue, 05 Mar 2019 11:26:37 -0800 (PST) Subject: [pypy-commit] pypy newmemoryview-app-level: mrege default into branch Message-ID: <5c7ecd6d.1c69fb81.420fb.2a7b@mx.google.com> Author: Matti Picus Branch: newmemoryview-app-level Changeset: r96212:c3596fa33c20 Date: 2019-03-05 21:25 +0200 http://bitbucket.org/pypy/pypy/changeset/c3596fa33c20/ Log: mrege default into branch diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst --- a/pypy/doc/__pypy__-module.rst +++ b/pypy/doc/__pypy__-module.rst @@ -1,25 +1,29 @@ -.. comment: this document is very incomplete, should we generate - it automatically? +.. comment: this document may get out of synch with the code, but to generate + it automatically we would need to use pypy to run sphinx-build The ``__pypy__`` module ======================= The ``__pypy__`` module is the main entry point to special features provided -by PyPy's standard interpreter. Its content depends on :doc:`configuration options ` -which may add new functionality and functions whose existence or non-existence -indicates the presence of such features. +by PyPy's standard interpreter. Its content depends on :doc:`configuration +options ` which may add new functionality and functions whose +existence or non-existence indicates the presence of such features. These are +generally used for compatibility when writing pure python modules that in +CPython are written in C. Not available in CPython, and so must be used inside a +``if platform.python_implementation == 'PyPy'`` block or otherwise hidden from +the CPython interpreter. Generally available functionality --------------------------------- - - ``internal_repr(obj)``: return the interpreter-level representation of an - object. - - ``bytebuffer(length)``: return a new read-write buffer of the given length. - It works like a simplified array of characters (actually, depending on the - configuration the ``array`` module internally uses this). + - ``internal_repr(obj)``: return the interpreter-level representation of an + object. + - ``bytebuffer(length)``: return a new read-write buffer of the given length. + It works like a simplified array of characters (actually, depending on the + configuration the ``array`` module internally uses this). - - ``attach_gdb()``: start a GDB at the interpreter-level (or a PDB before translation). + - ``attach_gdb()``: start a GDB at the interpreter-level (or a PDB before translation). - ``newmemoryview(buffer, itemsize, format, shape=None, strides=None)``: create a `memoryview` instance with the data from ``buffer`` and the @@ -29,6 +33,108 @@ ``__buffer__(self, flags)`` method. This method should return a memoryview instance of the class instance. It is called by the C-API's ``tp_as_buffer. bf_getbuffer``. + + - ``debug_stop`` + - ``debug_print`` + - ``debug_print_once`` + - ``debug_flush`` + - ``debug_read_timestamp`` + - ``debug_get_timestamp_unit`` + + - ``builtinify(func)``: To implement at app-level modules that are, in CPython, + implemented in C: this decorator protects a function from being ever bound + like a method. Useful because some tests do things like put a "built-in" + function on a class and access it via the instance. + + - ``hidden_applevel(func)``: Decorator that hides a function's frame from + app-level + + - ``get_hidden_tb()``: Return the traceback of the current exception being + handled by a frame hidden from applevel. + + - ``lookup_special(obj, meth)``: Lookup up a special method on an object. + - ``do_what_I_mean`` + + - ``resizelist_hint(...)``: Reallocate the underlying storage of the argument + list to sizehint + + - ``newlist_hint(...)``: Create a new empty list that has an underlying + storage of length sizehint + + - ``add_memory_pressure(bytes)``: Add memory pressure of estimate bytes. + Useful when calling a C function that internally allocates a big chunk of + memory. This instructs the GC to garbage collect sooner than it would + otherwise. + + - ``newdict(type)``: Create a normal dict with a special implementation + strategy. ``type`` is a string and can be: + + * ``"module"`` - equivalent to ``some_module.__dict__`` + + * ``"instance"`` - equivalent to an instance dict with a not-changing-much + set of keys + + * ``"kwargs"`` - keyword args dict equivalent of what you get from + ``**kwargs`` in a function, optimized for passing around + + * ``"strdict"`` - string-key only dict. This one should be chosen + automatically + + - ``reversed_dict``: Enumerate the keys in a dictionary object in reversed + order. This is a ``__pypy__`` function instead of being simply done by + calling reversed(), for CPython compatibility: dictionaries are ordered in + PyPY but not in Cpython2.7. You should use the collections.OrderedDict + class for cases where ordering is important. That class implements + ``__reversed__`` by calling __pypy__.reversed_dict() + + - ``dict_popitem_first``: Interp-level implementation of + ``OrderedDict.popitem(last=False)``. + + - ``delitem_if_value_is`` Atomic equivalent to: ``if dict.get(key) is value: + del dict[key]``. + + SPECIAL USE CASES ONLY! Avoid using on dicts which are specialized, + e.g. to ``int`` or ``str`` keys, because it switches to the object + strategy. Also, the ``is`` operation is really pointer equality, so avoid + using it if ``value`` is an immutable object like ``int`` or ``str``. + + - ``move_to_end``: Move the key in a dictionary object into the first or last + position. This is used in Python 3.x to implement ``OrderedDict.move_to_end()``. + + - ``strategy(dict or list or set)``: Return the underlying strategy currently + used by the object + + - ``specialized_zip_2_lists`` + - ``locals_to_fast`` + - ``set_code_callback`` + - ``save_module_content_for_future_reload`` + - ``decode_long`` + - ``side_effects_ok``: For use with the reverse-debugger: this function + normally returns True, but will return False if we are evaluating a + debugging command like a watchpoint. You are responsible for not doing any + side effect at all (including no caching) when evaluating watchpoints. This + function is meant to help a bit---you can write:: + + if not __pypy__.side_effects_ok(): + skip the caching logic + + inside getter methods or properties, to make them usable from + watchpoints. Note that you need to re-run ``REVDB=.. pypy`` + after changing the Python code. + + - ``stack_almost_full``: Return True if the stack is more than 15/16th full. + - ``pyos_inputhook``: Call PyOS_InputHook() from the CPython C API + - ``os.real_getenv(...)`` gets OS environment variables skipping python code + - ``_pypydatetime`` provides base classes with correct C API interactions for + the pure-python ``datetime`` stdlib module + +Fast String Concatenation +------------------------- +Rather than in-place concatenation ``+=``, use these to enable fast, minimal +copy, string building. + + - ``builders.StringBuilder`` + - ``builders.UnicodeBuilder`` Transparent Proxy Functionality ------------------------------- @@ -43,6 +149,30 @@ its controller. Otherwise return None. +Additional Clocks for Timing +---------------------------- +The ``time`` submodule exposes the platform-dependent clock types such as +``CLOCK_BOOTTIME``, ``CLOCK_MONOTONIC``, ``CLOCK_MONOTONIC_COARSE``, +``CLOCK_MONOTONIC_RAW`` and two functions: + + - ``clock_gettime(m)`` which returns the clock type time in seconds and + - ``clock_getres(m)`` which returns the clock resolution in seconds. + +Extended Signal Handling +------------------------ +``thread.signals_enbaled`` is a context manager to use in non-main threads. + enables receiving signals in a "with" statement. More precisely, if a + signal is received by the process, then the signal handler might be + called either in the main thread (as usual) or within another thread + that is within a "with signals_enabled:". This other thread should be + ready to handle unexpected exceptions that the signal handler might + raise --- notably KeyboardInterrupt. + +Integer Operations with Overflow +-------------------------------- + - ``intop`` provides a module with integer operations that have + two-complement overflow behaviour instead of overflowing to longs + Functionality available on py.py (not after translation) -------------------------------------------------------- From pypy.commits at gmail.com Tue Mar 5 14:53:44 2019 From: pypy.commits at gmail.com (rlamy) Date: Tue, 05 Mar 2019 11:53:44 -0800 (PST) Subject: [pypy-commit] pypy issue2968: Add test for #2968 Message-ID: <5c7ed3c8.1c69fb81.58dfc.1871@mx.google.com> Author: Ronan Lamy Branch: issue2968 Changeset: r96213:a0245fd17c4f Date: 2019-03-05 19:52 +0000 http://bitbucket.org/pypy/pypy/changeset/a0245fd17c4f/ Log: Add test for #2968 diff --git a/pypy/module/cpyext/test/THPSize.c b/pypy/module/cpyext/test/THPSize.c new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/test/THPSize.c @@ -0,0 +1,85 @@ +#include "Python.h" + +struct THPSize { + PyTupleObject tuple; +} THPSize; + +static PyObject * THPSize_pynew(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + return PyTuple_Type.tp_new(type, args, kwargs); +} + +static PyMappingMethods THPSize_as_mapping = { + 0, //PyTuple_Type.tp_as_mapping->mp_length, + 0, + 0 +}; + + +PyTypeObject THPSizeType = { + PyVarObject_HEAD_INIT(0, 0) + "torch.Size", /* tp_name */ + sizeof(THPSize), /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + &THPSize_as_mapping, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + &PyTuple_Type, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + THPSize_pynew, /* tp_new */ +}; + +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "THPSize", + "Module Doc", + -1, + NULL, + NULL, + NULL, + NULL, + NULL, +}; + +PyMODINIT_FUNC +PyInit_THPSize(void) +{ + PyObject *module = PyModule_Create(&moduledef); + THPSize_as_mapping.mp_length = PyTuple_Type.tp_as_mapping->mp_length; + if (PyType_Ready(&THPSizeType) < 0) { + return NULL; + } + Py_INCREF(&THPSizeType); + if (PyModule_AddObject(module, "Size", (PyObject*)&THPSizeType) < 0) { + return NULL; + } + return module; +} diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py --- a/pypy/module/cpyext/test/test_tupleobject.py +++ b/pypy/module/cpyext/test/test_tupleobject.py @@ -226,3 +226,7 @@ raises(SystemError, module.set_after_use, s) else: module.set_after_use(s) + + def test_torch(self): + module = self.import_module('THPSize') + module.Size() From pypy.commits at gmail.com Wed Mar 6 05:33:01 2019 From: pypy.commits at gmail.com (fijal) Date: Wed, 06 Mar 2019 02:33:01 -0800 (PST) Subject: [pypy-commit] pypy arm64: enough scaffolding to actually run the first loop test. It does not pass yet, but it does run! Message-ID: <5c7fa1dd.1c69fb81.c3158.771a@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96214:173ed1a7572d Date: 2019-03-06 10:32 +0000 http://bitbucket.org/pypy/pypy/changeset/173ed1a7572d/ Log: enough scaffolding to actually run the first loop test. It does not pass yet, but it does run! diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -9,6 +9,7 @@ # CoreRegisterManager, check_imm_arg, VFPRegisterManager, #from rpython.jit.backend.arm import callbuilder from rpython.jit.backend.aarch64 import registers as r +from rpython.jit.backend.arm import conditions as c from rpython.jit.backend.llsupport import jitframe from rpython.jit.backend.llsupport.assembler import BaseAssembler from rpython.jit.backend.llsupport.regalloc import get_scale, valid_addressing_size @@ -113,6 +114,7 @@ def setup(self, looptoken): BaseAssembler.setup(self, looptoken) + self.failure_recovery_code = [0, 0, 0, 0] assert self.memcpy_addr != 0, 'setup_once() not called?' if we_are_translated(): self.debug = False @@ -135,7 +137,48 @@ self.pending_guards = None def _build_failure_recovery(self, exc, withfloats=False): - pass # XXX + return # XXX + mc = InstrBuilder() + self._push_all_regs_to_jitframe(mc, [], withfloats) + + if exc: + XXX + # We might have an exception pending. Load it into r4 + # (this is a register saved across calls) + mc.gen_load_int(r.r5.value, self.cpu.pos_exc_value()) + mc.LDR_ri(r.r4.value, r.r5.value) + # clear the exc flags + mc.gen_load_int(r.r6.value, 0) + mc.STR_ri(r.r6.value, r.r5.value) # pos_exc_value is still in r5 + mc.gen_load_int(r.r5.value, self.cpu.pos_exception()) + mc.STR_ri(r.r6.value, r.r5.value) + # save r4 into 'jf_guard_exc' + offset = self.cpu.get_ofs_of_frame_field('jf_guard_exc') + assert check_imm_arg(abs(offset)) + mc.STR_ri(r.r4.value, r.fp.value, imm=offset) + # now we return from the complete frame, which starts from + # _call_header_with_stack_check(). The LEA in _call_footer below + # throws away most of the frame, including all the PUSHes that we + # did just above. + ofs = self.cpu.get_ofs_of_frame_field('jf_descr') + assert check_imm_arg(abs(ofs)) + ofs2 = self.cpu.get_ofs_of_frame_field('jf_gcmap') + assert check_imm_arg(abs(ofs2)) + base_ofs = self.cpu.get_baseofs_of_frame_field() + # store the gcmap + mc.POP([r.ip.value]) + mc.STR_ri(r.ip.value, r.fp.value, imm=ofs2) + # store the descr + mc.POP([r.ip.value]) + mc.STR_ri(r.ip.value, r.fp.value, imm=ofs) + + # set return value + assert check_imm_arg(base_ofs) + mc.MOV_rr(r.r0.value, r.fp.value) + # + self.gen_func_epilog(mc) + rawstart = mc.materialize(self.cpu, []) + self.failure_recovery_code[exc + 2 * withfloats] = rawstart def _build_wb_slowpath(self, withcards, withfloats=False, for_frame=False): pass # XXX @@ -159,8 +202,25 @@ baseofs = self.cpu.get_baseofs_of_frame_field() self.current_clt.frame_info.update_frame_depth(baseofs, frame_depth) + def generate_quick_failure(self, guardtok): + startpos = self.mc.currpos() + faildescrindex, target = self.store_info_on_descr(startpos, guardtok) + self.mc.SUB_ri(r.sp.value, r.sp.value, 2 * WORD) + self.load_from_gc_table(r.ip0.value, faildescrindex) + self.store_reg(self.mc, r.ip0, r.fp, WORD) + self.push_gcmap(self.mc, gcmap=guardtok.gcmap, ofs=0) + self.mc.BL(target) + return startpos + + def push_gcmap(self, mc, gcmap, ofs): + ptr = rffi.cast(lltype.Signed, gcmap) + mc.gen_load_int(r.ip0.value, ptr) + self.store_reg(mc, r.ip0, r.fp, ofs) + def write_pending_failure_recoveries(self): - pass # XXX + for tok in self.pending_guards: + #generate the exit stub and the encoded representation + tok.pos_recovery_stub = self.generate_quick_failure(tok) def reserve_gcref_table(self, allgcrefs): gcref_table_size = len(allgcrefs) * WORD @@ -200,8 +260,25 @@ # self.codemap.get_final_bytecode(res, size)) return res - def process_pending_guards(self, rawstart): - pass + def process_pending_guards(self, block_start): + clt = self.current_clt + for tok in self.pending_guards: + descr = tok.faildescr + assert isinstance(descr, AbstractFailDescr) + failure_recovery_pos = block_start + tok.pos_recovery_stub + descr.adr_jump_offset = failure_recovery_pos + relative_offset = tok.pos_recovery_stub - tok.offset + guard_pos = block_start + tok.offset + if not tok.guard_not_invalidated(): + # patch the guard jump to the stub + # overwrite the generate BRK with a B_offs to the pos of the + # stub + mc = InstrBuilder() + mc.B_ofs_cond(relative_offset, c.get_opposite_of(tok.fcond)) + mc.copy_to_raw_memory(guard_pos) + else: + XX + clt.invalidate_positions.append((guard_pos, relative_offset)) def fixup_target_tokens(self, rawstart): for targettoken in self.target_tokens_currently_compiling: @@ -277,10 +354,11 @@ elif not we_are_translated() and op.getopnum() == rop.FORCE_SPILL: regalloc.prepare_force_spill(op) elif i < len(operations) - 1 and regalloc.next_op_can_accept_cc(operations, i): - arglocs = guard_operations[operations[i + 1].getopnum()]( - regalloc, operations[i + 1], op) + guard_op = operations[i + 1] + guard_num = guard_op.getopnum() + arglocs, fcond = guard_operations[guard_num](regalloc, guard_op, op) if arglocs is not None: - xxx + asm_guard_operations[guard_num](self, guard_op, fcond, arglocs) regalloc.next_instruction() # advance one more else: arglocs = regalloc_operations[opnum](regalloc, op) @@ -302,8 +380,7 @@ opnum = op.getopnum() arglocs = comp_operations[opnum](self._regalloc, op, True) assert arglocs is not None - asm_comp_operations[opnum](self, op, arglocs) - return arglocs + return asm_comp_operations[opnum](self, op, arglocs) # regalloc support def load(self, loc, value): @@ -353,6 +430,14 @@ # if save_helper: # self.mc.POP([helper.value], cond=cond) + def _mov_reg_to_loc(self, prev_loc, loc): + if loc.is_core_reg(): + self.mc.MOV_rr(loc.value, prev_loc.value) + elif loc.is_stack(): + self.mc.STR_ri(r.fp.value, prev_loc.value, loc.value) + else: + XXX + def regalloc_mov(self, prev_loc, loc): """Moves a value from a previous location to some other location""" if prev_loc.is_imm(): @@ -420,6 +505,18 @@ # mc.gen_load_int(r.ip1, ofs) # mc.STR_rr(source.value, base.value, r.ip1) + def check_frame_before_jump(self, target_token): + if target_token in self.target_tokens_currently_compiling: + return + if target_token._arm_clt is self.current_clt: + return + # We can have a frame coming from god knows where that's + # passed to a jump to another loop. Make sure it has the + # correct depth + expected_size = target_token._arm_clt.frame_info.jfi_frame_depth + self._check_frame_depth(self.mc, self._regalloc.get_gcmap(), + expected_size=expected_size) + def not_implemented(msg): msg = '[ARM/asm] %s\n' % msg @@ -436,7 +533,12 @@ print "[ARM/asm] %s not implemented" % op.getopname() raise NotImplementedError(op) +def notimplemented_guard_op(self, op, fcond, arglocs): + print "[ARM/asm] %s not implemented" % op.getopname() + raise NotImplementedError(op) + asm_operations = [notimplemented_op] * (rop._LAST + 1) +asm_guard_operations = [notimplemented_guard_op] * (rop._LAST + 1) asm_comp_operations = [notimplemented_comp_op] * (rop._LAST + 1) asm_extra_operations = {} @@ -449,6 +551,10 @@ opname = name[len('emit_op_'):] num = getattr(rop, opname.upper()) asm_operations[num] = value + elif name.startswith('emit_guard_op_'): + opname = name[len('emit_guard_op_'):] + num = getattr(rop, opname.upper()) + asm_guard_operations[num] = value elif name.startswith('emit_comp_op_'): opname = name[len('emit_comp_op_'):] num = getattr(rop, opname.upper()) diff --git a/rpython/jit/backend/aarch64/codebuilder.py b/rpython/jit/backend/aarch64/codebuilder.py --- a/rpython/jit/backend/aarch64/codebuilder.py +++ b/rpython/jit/backend/aarch64/codebuilder.py @@ -7,6 +7,7 @@ from rpython.rtyper.lltypesystem import lltype, rffi from rpython.tool.udir import udir +PC_OFFSET = 8 class AbstractAarch64Builder(object): def write32(self, word): @@ -70,6 +71,11 @@ self.write32((base << 22) | (constant << 10) | (rn << 5) | rd) + def SUB_ri(self, rd, rn, constant): + base = 0b1101000100 + assert 0 <= constant < 4096 + self.write32((base << 22) | (constant << 10) | (rn << 5) | rd) + def LDP_rri(self, reg1, reg2, rn, offset): base = 0b1010100101 assert -512 <= offset < 512 @@ -108,6 +114,34 @@ base = 0b11101011000 self.write32((base << 21) | (rm << 16) | (rn << 5) | 0b11111) + def B_ofs(self, ofs): + base = 0b000101 + assert ofs & 0x3 == 0 + pos = self.currpos() + target_ofs = ofs - (pos + PC_OFFSET) + assert -(1 << (26 + 2)) < target_ofs < 1<<(26 + 2) + if target_ofs < 0: + target_ofs = 1<<25 | (~target_ofs) + self.write32((base << 26) | (target_ofs >> 2)) + + def B_ofs_cond(self, ofs, cond): + base = 0b01010100 + assert ofs & 0x3 == 0 + assert -1 << 10 < ofs < 1 << 10 + imm = ofs >> 2 + if imm < 0: + xxx + self.write32((base << 24) | (imm << 5) | cond) + + def BL(self, target): + target = rffi.cast(lltype.Signed, target) + self.gen_load_int(r.ip0.value, target) + self.BR(r.ip0.value) + + def BR(self, reg): + base = 0b1101011000011111000000 + self.write32((base << 10) | (reg << 5)) + def BRK(self): self.write32(0b11010100001 << 21) @@ -116,9 +150,12 @@ register""" # XXX optimize! self.MOVZ_r_u16(r, value & 0xFFFF, 0) - self.MOVK_r_u16(r, (value >> 16) & 0xFFFF, 16) - self.MOVK_r_u16(r, (value >> 32) & 0xFFFF, 32) - self.MOVK_r_u16(r, (value >> 48) & 0xFFFF, 48) + value = value >> 16 + shift = 16 + while value: + self.MOVK_r_u16(r, (value >> 16) & 0xFFFF, shift) + shift += 16 + value >>= 16 class InstrBuilder(BlockBuilderMixin, AbstractAarch64Builder): diff --git a/rpython/jit/backend/aarch64/locations.py b/rpython/jit/backend/aarch64/locations.py --- a/rpython/jit/backend/aarch64/locations.py +++ b/rpython/jit/backend/aarch64/locations.py @@ -47,7 +47,6 @@ return True def as_key(self): # 0 <= as_key <= 30, 31 being zero register - xxx return self.value class VFPRegisterLocation(RegisterLocation): @@ -64,7 +63,6 @@ return True def as_key(self): # 40 <= as_key <= 71 - xxx return self.value + 40 def is_float(self): @@ -110,7 +108,6 @@ return True def as_key(self): # an aligned word + 10000 - XXX return self.position + 10000 def is_float(self): @@ -127,7 +124,10 @@ return "xzr" def as_key(self): - return 31 + raise ValueError("should never make it to jump") + +def imm(i): + return ImmLocation(i) def get_fp_offset(base_ofs, position): return base_ofs + WORD * (position + JITFRAME_FIXED_SIZE) diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -2,8 +2,12 @@ from rpython.jit.metainterp.history import (AbstractFailDescr, ConstInt, INT, FLOAT, REF) from rpython.jit.backend.aarch64 import registers as r -from rpython.jit.backend.arm import conditions as c # yes, arm, not aarch64 +from rpython.jit.backend.arm import conditions as c +from rpython.jit.backend.aarch64.arch import JITFRAME_FIXED_SIZE from rpython.jit.backend.llsupport.assembler import GuardToken, BaseAssembler +from rpython.jit.backend.llsupport.gcmap import allocate_gcmap +from rpython.jit.metainterp.history import TargetToken + class ResOpAssembler(BaseAssembler): def emit_op_int_add(self, op, arglocs): @@ -34,7 +38,13 @@ else: self.mc.CMP_rr(l0.value, l1.value) - emit_comp_op_int_le = emit_int_comp_op + def emit_comp_op_int_lt(self, op, arglocs): + self.emit_int_comp_op(op, arglocs) + return c.LT + + def emit_comp_op_int_le(self, op, arglocs): + self.emit_int_comp_op(op, arglocs) + return c.LE def emit_op_increment_debug_counter(self, op, arglocs): return # XXXX @@ -43,9 +53,52 @@ self.mc.ADD_ri(value_loc.value, value_loc.value, 1) self.mc.STR_ri(value_loc.value, base_loc.value, 0) + def build_guard_token(self, op, frame_depth, arglocs, offset, fcond): + descr = op.getdescr() + assert isinstance(descr, AbstractFailDescr) + + gcmap = allocate_gcmap(self, frame_depth, JITFRAME_FIXED_SIZE) + faildescrindex = self.get_gcref_from_faildescr(descr) + token = GuardToken(self.cpu, gcmap, descr, + failargs=op.getfailargs(), + fail_locs=arglocs, + guard_opnum=op.getopnum(), + frame_depth=frame_depth, + faildescrindex=faildescrindex) + token.fcond = fcond + return token + + def _emit_guard(self, op, fcond, arglocs, is_guard_not_invalidated=False): + pos = self.mc.currpos() + token = self.build_guard_token(op, arglocs[0].value, arglocs[1:], pos, + fcond) + token.offset = pos + self.pending_guards.append(token) + assert token.guard_not_invalidated() == is_guard_not_invalidated + # For all guards that are not GUARD_NOT_INVALIDATED we emit a + # breakpoint to ensure the location is patched correctly. In the case + # of GUARD_NOT_INVALIDATED we use just a NOP, because it is only + # eventually patched at a later point. + if is_guard_not_invalidated: + self.mc.NOP() + else: + self.mc.BRK() + + def emit_guard_op_guard_true(self, guard_op, fcond, arglocs): + self._emit_guard(guard_op, fcond, arglocs) + def emit_op_label(self, op, arglocs): pass + def emit_op_jump(self, op, arglocs): + target_token = op.getdescr() + assert isinstance(target_token, TargetToken) + target = target_token._ll_loop_code + if target_token in self.target_tokens_currently_compiling: + self.mc.B_ofs(target) + else: + self.mc.B(target) + def emit_op_finish(self, op, arglocs): base_ofs = self.cpu.get_baseofs_of_frame_field() if len(arglocs) > 0: diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -12,6 +12,8 @@ get_scale from rpython.rtyper.lltypesystem import lltype, rffi, rstr, llmemory from rpython.jit.backend.aarch64 import registers as r +from rpython.jit.backend.arm.jump import remap_frame_layout_mixed +from rpython.jit.backend.aarch64.locations import imm class TempInt(TempVar): @@ -363,11 +365,54 @@ return locs def prepare_guard_op_guard_true(self, op, prevop): - arglocs = self.assembler.dispatch_comparison(prevop) - xxx + fcond = self.assembler.dispatch_comparison(prevop) + # result is in CC + + arglocs = [None] * (len(op.getfailargs()) + 1) + arglocs[0] = imm(self.frame_manager.get_frame_depth()) + failargs = op.getfailargs() + for i in range(len(failargs)): + if failargs[i]: + arglocs[i + 1] = self.loc(failargs[i]) + return arglocs, fcond prepare_op_nursery_ptr_increment = prepare_op_int_add + def prepare_op_jump(self, op): + assert self.jump_target_descr is None + descr = op.getdescr() + assert isinstance(descr, TargetToken) + self.jump_target_descr = descr + arglocs = descr._arm_arglocs + + # get temporary locs + tmploc = r.ip0 + vfptmploc = None # XXX r.vfp_ip + + # Part about non-floats + src_locations1 = [] + dst_locations1 = [] + # Part about floats + src_locations2 = [] + dst_locations2 = [] + + # Build the four lists + for i in range(op.numargs()): + box = op.getarg(i) + src_loc = self.loc(box) + dst_loc = arglocs[i] + if box.type != FLOAT: + src_locations1.append(src_loc) + dst_locations1.append(dst_loc) + else: + src_locations2.append(src_loc) + dst_locations2.append(dst_loc) + self.assembler.check_frame_before_jump(self.jump_target_descr) + remap_frame_layout_mixed(self.assembler, + src_locations1, dst_locations1, tmploc, + src_locations2, dst_locations2, vfptmploc) + return [] + def force_allocate_reg(self, var, forbidden_vars=[], selected_reg=None): if var.type == FLOAT: return self.vfprm.force_allocate_reg(var, forbidden_vars, diff --git a/rpython/jit/backend/aarch64/runner.py b/rpython/jit/backend/aarch64/runner.py --- a/rpython/jit/backend/aarch64/runner.py +++ b/rpython/jit/backend/aarch64/runner.py @@ -1,11 +1,14 @@ from rpython.rtyper.lltypesystem import llmemory, lltype from rpython.jit.backend.aarch64.assembler import AssemblerARM64 +from rpython.jit.backend.aarch64 import registers as r from rpython.jit.backend.llsupport.llmodel import AbstractLLCPU class CPU_ARM64(AbstractLLCPU): """ARM 64""" backend_name = "aarch64" + frame_reg = r.fp + all_reg_indexes = range(len(r.all_regs)) IS_64_BIT = True diff --git a/rpython/jit/backend/aarch64/test/test_instr_builder.py b/rpython/jit/backend/aarch64/test/test_instr_builder.py --- a/rpython/jit/backend/aarch64/test/test_instr_builder.py +++ b/rpython/jit/backend/aarch64/test/test_instr_builder.py @@ -11,6 +11,9 @@ def writechar(self, char): self.buffer.append(char) + def currpos(self): + return 0 + def hexdump(self): return ''.join(self.buffer) @@ -125,6 +128,15 @@ assert cb.hexdump() == assemble("ADD %r, %r, %r" % (rd, rn, rm)) @settings(max_examples=20) + @given(rd=st.sampled_from(r.registers), + rn=st.sampled_from(r.registers), + ofs=st.integers(min_value=0, max_value=4095)) + def test_SUB_ri(self, rd, rn, ofs): + cb = CodeBuilder() + cb.SUB_ri(rd.value, rn.value, ofs) + assert cb.hexdump() == assemble("SUB %r, %r, %d" % (rd, rn, ofs)) + + @settings(max_examples=20) @given(rn=st.sampled_from(r.registers), rm=st.sampled_from(r.registers)) def test_CMP_rr(self, rn, rm): diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -185,6 +185,11 @@ """, namespace={'targettoken': targettoken, 'fdescr': BasicFailDescr(2)}) self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) + print "ONE" + deadframe = self.cpu.execute_token(looptoken, 10) + print "TWO" + fail = self.cpu.get_latest_descr(deadframe) + assert fail.identifier == 2 deadframe = self.cpu.execute_token(looptoken, 2) fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 2 From pypy.commits at gmail.com Wed Mar 6 07:09:01 2019 From: pypy.commits at gmail.com (fijal) Date: Wed, 06 Mar 2019 04:09:01 -0800 (PST) Subject: [pypy-commit] pypy arm64: enough to successfully exit the loop Message-ID: <5c7fb85d.1c69fb81.cf3e4.861b@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96215:91919b103009 Date: 2019-03-06 12:08 +0000 http://bitbucket.org/pypy/pypy/changeset/91919b103009/ Log: enough to successfully exit the loop diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -5,7 +5,8 @@ #from rpython.jit.backend.arm.helper.regalloc import VMEM_imm_size from rpython.jit.backend.aarch64.opassembler import ResOpAssembler from rpython.jit.backend.aarch64.regalloc import (Regalloc, - operations as regalloc_operations, guard_operations, comp_operations) + operations as regalloc_operations, guard_operations, comp_operations, + CoreRegisterManager) # CoreRegisterManager, check_imm_arg, VFPRegisterManager, #from rpython.jit.backend.arm import callbuilder from rpython.jit.backend.aarch64 import registers as r @@ -28,6 +29,10 @@ from rpython.rlib.rjitlog import rjitlog as jl class AssemblerARM64(ResOpAssembler): + def __init__(self, cpu, translate_support_code=False): + ResOpAssembler.__init__(self, cpu, translate_support_code) + self.failure_recovery_code = [0, 0, 0, 0] + def assemble_loop(self, jd_id, unique_id, logger, loopname, inputargs, operations, looptoken, log): clt = CompiledLoopToken(self.cpu, looptoken.number) @@ -114,7 +119,6 @@ def setup(self, looptoken): BaseAssembler.setup(self, looptoken) - self.failure_recovery_code = [0, 0, 0, 0] assert self.memcpy_addr != 0, 'setup_once() not called?' if we_are_translated(): self.debug = False @@ -136,12 +140,43 @@ self.mc = None self.pending_guards = None + def _push_all_regs_to_jitframe(self, mc, ignored_regs, withfloats, + callee_only=False): + # Push general purpose registers + base_ofs = self.cpu.get_baseofs_of_frame_field() + if callee_only: + regs = CoreRegisterManager.save_around_call_regs + else: + regs = CoreRegisterManager.all_regs + # XXX add special case if ignored_regs are a block at the start of regs + if not ignored_regs: # we want to push a contiguous block of regs + assert base_ofs < 0x100 + mc.ADD_ri(r.ip0.value, r.fp.value, base_ofs) + # should explode the test + #mc.STM(r.ip.value, [reg.value for reg in regs]) + else: + for reg in ignored_regs: + assert not reg.is_vfp_reg() # sanity check + # we can have holes in the list of regs + for i, gpr in enumerate(regs): + if gpr in ignored_regs: + continue + self.store_reg(mc, gpr, r.fp, base_ofs + i * WORD) + + if withfloats: + # Push VFP regs + regs = VFPRegisterManager.all_regs + ofs = len(CoreRegisterManager.all_regs) * WORD + assert check_imm_arg(ofs+base_ofs) + mc.ADD_ri(r.ip.value, r.fp.value, imm=ofs+base_ofs) + mc.VSTM(r.ip.value, [vfpr.value for vfpr in regs]) + def _build_failure_recovery(self, exc, withfloats=False): - return # XXX mc = InstrBuilder() self._push_all_regs_to_jitframe(mc, [], withfloats) if exc: + return # fix later XXX # We might have an exception pending. Load it into r4 # (this is a register saved across calls) @@ -161,20 +196,20 @@ # throws away most of the frame, including all the PUSHes that we # did just above. ofs = self.cpu.get_ofs_of_frame_field('jf_descr') - assert check_imm_arg(abs(ofs)) + assert ofs <= 0x100 ofs2 = self.cpu.get_ofs_of_frame_field('jf_gcmap') - assert check_imm_arg(abs(ofs2)) - base_ofs = self.cpu.get_baseofs_of_frame_field() + assert ofs2 <= 0x100 # store the gcmap - mc.POP([r.ip.value]) - mc.STR_ri(r.ip.value, r.fp.value, imm=ofs2) + mc.LDR_ri(r.ip0.value, r.sp.value, 0) + mc.STR_ri(r.ip0.value, r.fp.value, ofs2) # store the descr - mc.POP([r.ip.value]) - mc.STR_ri(r.ip.value, r.fp.value, imm=ofs) + mc.LDR_ri(r.ip0.value, r.sp.value, WORD) + mc.STR_ri(r.ip0.value, r.fp.value, ofs) # set return value - assert check_imm_arg(base_ofs) - mc.MOV_rr(r.r0.value, r.fp.value) + mc.MOV_rr(r.x0.value, r.fp.value) + # move the stack + mc.ADD_ri(r.sp.value, r.sp.value, 2 * WORD) # self.gen_func_epilog(mc) rawstart = mc.materialize(self.cpu, []) @@ -434,7 +469,7 @@ if loc.is_core_reg(): self.mc.MOV_rr(loc.value, prev_loc.value) elif loc.is_stack(): - self.mc.STR_ri(r.fp.value, prev_loc.value, loc.value) + self.mc.STR_ri(prev_loc.value, r.fp.value, loc.value) else: XXX diff --git a/rpython/jit/backend/aarch64/codebuilder.py b/rpython/jit/backend/aarch64/codebuilder.py --- a/rpython/jit/backend/aarch64/codebuilder.py +++ b/rpython/jit/backend/aarch64/codebuilder.py @@ -153,7 +153,7 @@ value = value >> 16 shift = 16 while value: - self.MOVK_r_u16(r, (value >> 16) & 0xFFFF, shift) + self.MOVK_r_u16(r, value & 0xFFFF, shift) shift += 16 value >>= 16 From pypy.commits at gmail.com Wed Mar 6 08:35:24 2019 From: pypy.commits at gmail.com (fijal) Date: Wed, 06 Mar 2019 05:35:24 -0800 (PST) Subject: [pypy-commit] pypy arm64: pass the first loop test! Message-ID: <5c7fcc9c.1c69fb81.32f27.b97a@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96216:144deb45999c Date: 2019-03-06 13:34 +0000 http://bitbucket.org/pypy/pypy/changeset/144deb45999c/ Log: pass the first loop test! diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -151,9 +151,8 @@ # XXX add special case if ignored_regs are a block at the start of regs if not ignored_regs: # we want to push a contiguous block of regs assert base_ofs < 0x100 - mc.ADD_ri(r.ip0.value, r.fp.value, base_ofs) - # should explode the test - #mc.STM(r.ip.value, [reg.value for reg in regs]) + for i, reg in enumerate(regs): + mc.STR_ri(reg.value, r.fp.value, base_ofs + i * WORD) else: for reg in ignored_regs: assert not reg.is_vfp_reg() # sanity check @@ -195,22 +194,10 @@ # _call_header_with_stack_check(). The LEA in _call_footer below # throws away most of the frame, including all the PUSHes that we # did just above. - ofs = self.cpu.get_ofs_of_frame_field('jf_descr') - assert ofs <= 0x100 - ofs2 = self.cpu.get_ofs_of_frame_field('jf_gcmap') - assert ofs2 <= 0x100 - # store the gcmap - mc.LDR_ri(r.ip0.value, r.sp.value, 0) - mc.STR_ri(r.ip0.value, r.fp.value, ofs2) - # store the descr - mc.LDR_ri(r.ip0.value, r.sp.value, WORD) - mc.STR_ri(r.ip0.value, r.fp.value, ofs) # set return value mc.MOV_rr(r.x0.value, r.fp.value) - # move the stack - mc.ADD_ri(r.sp.value, r.sp.value, 2 * WORD) - # + self.gen_func_epilog(mc) rawstart = mc.materialize(self.cpu, []) self.failure_recovery_code[exc + 2 * withfloats] = rawstart @@ -240,7 +227,6 @@ def generate_quick_failure(self, guardtok): startpos = self.mc.currpos() faildescrindex, target = self.store_info_on_descr(startpos, guardtok) - self.mc.SUB_ri(r.sp.value, r.sp.value, 2 * WORD) self.load_from_gc_table(r.ip0.value, faildescrindex) self.store_reg(self.mc, r.ip0, r.fp, WORD) self.push_gcmap(self.mc, gcmap=guardtok.gcmap, ofs=0) @@ -270,8 +256,8 @@ self.setup_gcrefs_list(allgcrefs) def patch_gcref_table(self, looptoken, rawstart): - # the gc table is at the start of the machine code - self.gc_table_addr = rawstart + # the gc table is at the start of the machine code + self.gc_table_addr = rawstart tracer = self.cpu.gc_ll_descr.make_gcref_tracer(rawstart, self._allgcrefs) gcreftracers = self.get_asmmemmgr_gcreftracers(looptoken) diff --git a/rpython/jit/backend/aarch64/codebuilder.py b/rpython/jit/backend/aarch64/codebuilder.py --- a/rpython/jit/backend/aarch64/codebuilder.py +++ b/rpython/jit/backend/aarch64/codebuilder.py @@ -7,7 +7,7 @@ from rpython.rtyper.lltypesystem import lltype, rffi from rpython.tool.udir import udir -PC_OFFSET = 8 +PC_OFFSET = 0 # XXX class AbstractAarch64Builder(object): def write32(self, word): @@ -121,8 +121,10 @@ target_ofs = ofs - (pos + PC_OFFSET) assert -(1 << (26 + 2)) < target_ofs < 1<<(26 + 2) if target_ofs < 0: - target_ofs = 1<<25 | (~target_ofs) - self.write32((base << 26) | (target_ofs >> 2)) + target_ofs = (1 << 26) - (-target_ofs >> 2) + else: + target_ofs = target_ofs >> 2 + self.write32((base << 26) | target_ofs) def B_ofs_cond(self, ofs, cond): base = 0b01010100 diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -185,9 +185,7 @@ """, namespace={'targettoken': targettoken, 'fdescr': BasicFailDescr(2)}) self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) - print "ONE" deadframe = self.cpu.execute_token(looptoken, 10) - print "TWO" fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 2 deadframe = self.cpu.execute_token(looptoken, 2) From pypy.commits at gmail.com Wed Mar 6 09:32:04 2019 From: pypy.commits at gmail.com (fijal) Date: Wed, 06 Mar 2019 06:32:04 -0800 (PST) Subject: [pypy-commit] pypy arm64: basic IMM support in add Message-ID: <5c7fd9e4.1c69fb81.8193.02cc@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96217:13d01ec04e4e Date: 2019-03-06 14:31 +0000 http://bitbucket.org/pypy/pypy/changeset/13d01ec04e4e/ Log: basic IMM support in add diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -23,9 +23,9 @@ else: s = 0 if l0.is_imm(): - self.mc.ADD_ri(res.value, l1.value, imm=l0.value, s=s) + self.mc.ADD_ri(res.value, l1.value, l0.value) elif l1.is_imm(): - self.mc.ADD_ri(res.value, l0.value, imm=l1.value, s=s) + self.mc.ADD_ri(res.value, l0.value, l1.value) else: self.mc.ADD_rr(res.value, l0.value, l1.value) diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -148,6 +148,22 @@ continue return free_regs[i] +DEFAULT_IMM_SIZE = 4096 + +def check_imm_arg(arg, size=DEFAULT_IMM_SIZE, allow_zero=True): + i = arg + if allow_zero: + lower_bound = i >= 0 + else: + lower_bound = i > 0 + return i <= size and lower_bound + +def check_imm_box(arg, size=DEFAULT_IMM_SIZE, allow_zero=True): + if isinstance(arg, ConstInt): + return check_imm_arg(arg.getint(), size, allow_zero) + return False + + class Regalloc(BaseRegalloc): def __init__(self, assembler): @@ -280,16 +296,24 @@ return [base_loc, value_loc] def prepare_op_int_add(self, op): - arg0 = op.getarg(0) - arg1 = op.getarg(1) + boxes = op.getarglist() + a0, a1 = boxes # XXX support immediates - l0 = self.make_sure_var_in_reg(arg0, op.getarglist()) - l1 = self.make_sure_var_in_reg(arg1, op.getarglist()) + imm_a0 = check_imm_box(a0) + imm_a1 = check_imm_box(a1) + if not imm_a0 and imm_a1: + l0 = self.make_sure_var_in_reg(a0, boxes) + l1 = self.convert_to_imm(a1) + elif imm_a0 and not imm_a1: + l0 = self.convert_to_imm(a0) + l1 = self.make_sure_var_in_reg(a1, boxes) + else: + l0 = self.make_sure_var_in_reg(a0, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) self.possibly_free_vars_for_op(op) res = self.force_allocate_reg(op) return [l0, l1, res] - def prepare_int_cmp(self, op, res_in_cc): boxes = op.getarglist() arg0, arg1 = boxes diff --git a/rpython/jit/backend/aarch64/test/test_instr_builder.py b/rpython/jit/backend/aarch64/test/test_instr_builder.py --- a/rpython/jit/backend/aarch64/test/test_instr_builder.py +++ b/rpython/jit/backend/aarch64/test/test_instr_builder.py @@ -130,6 +130,15 @@ @settings(max_examples=20) @given(rd=st.sampled_from(r.registers), rn=st.sampled_from(r.registers), + imm=st.integers(min_value=0, max_value=(1<<12)-1)) + def test_ADD_ri(self, rd, rn, imm): + cb = CodeBuilder() + cb.ADD_ri(rd.value, rn.value, imm) + assert cb.hexdump() == assemble("ADD %r, %r, %d" % (rd, rn, imm)) + + @settings(max_examples=20) + @given(rd=st.sampled_from(r.registers), + rn=st.sampled_from(r.registers), ofs=st.integers(min_value=0, max_value=4095)) def test_SUB_ri(self, rd, rn, ofs): cb = CodeBuilder() From pypy.commits at gmail.com Wed Mar 6 09:36:08 2019 From: pypy.commits at gmail.com (fijal) Date: Wed, 06 Mar 2019 06:36:08 -0800 (PST) Subject: [pypy-commit] pypy arm64: int_sub Message-ID: <5c7fdad8.1c69fb81.b7a07.0ced@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96218:88c9f1531c0d Date: 2019-03-06 14:35 +0000 http://bitbucket.org/pypy/pypy/changeset/88c9f1531c0d/ Log: int_sub diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -13,6 +13,29 @@ def emit_op_int_add(self, op, arglocs): return self.int_add_impl(op, arglocs) + def int_sub_impl(self, op, arglocs, flags=0): + l0, l1, res = arglocs + if flags: + xxx + s = 1 + else: + s = 0 + if l0.is_imm(): + value = l0.getint() + assert value >= 0 + # reverse substract ftw + XX + self.mc.RSB_ri(res.value, l1.value, value) + elif l1.is_imm(): + value = l1.getint() + assert value >= 0 + self.mc.SUB_ri(res.value, l0.value, value) + else: + self.mc.SUB_rr(res.value, l0.value, l1.value) + + def emit_op_int_sub(self, op, arglocs): + self.int_sub_impl(op, arglocs) + emit_op_nursery_ptr_increment = emit_op_int_add def int_add_impl(self, op, arglocs, ovfcheck=False): diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -298,7 +298,6 @@ def prepare_op_int_add(self, op): boxes = op.getarglist() a0, a1 = boxes - # XXX support immediates imm_a0 = check_imm_box(a0) imm_a1 = check_imm_box(a1) if not imm_a0 and imm_a1: @@ -314,6 +313,8 @@ res = self.force_allocate_reg(op) return [l0, l1, res] + prepare_op_int_sub = prepare_op_int_add + def prepare_int_cmp(self, op, res_in_cc): boxes = op.getarglist() arg0, arg1 = boxes From pypy.commits at gmail.com Wed Mar 6 10:05:14 2019 From: pypy.commits at gmail.com (fijal) Date: Wed, 06 Mar 2019 07:05:14 -0800 (PST) Subject: [pypy-commit] pypy arm64: bridge basics Message-ID: <5c7fe1aa.1c69fb81.1d92a.d90b@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96219:a50f3a71ef1d Date: 2019-03-06 14:49 +0000 http://bitbucket.org/pypy/pypy/changeset/a50f3a71ef1d/ Log: bridge basics diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -117,6 +117,89 @@ return AsmInfo(ops_offset, rawstart + loop_head, size_excluding_failure_stuff - loop_head) + def assemble_bridge(self, logger, faildescr, inputargs, operations, + original_loop_token, log): + if not we_are_translated(): + # Arguments should be unique + assert len(set(inputargs)) == len(inputargs) + + self.setup(original_loop_token) + #self.codemap.inherit_code_from_position(faildescr.adr_jump_offset) + descr_number = compute_unique_id(faildescr) + if log: + operations = self._inject_debugging_code(faildescr, operations, + 'b', descr_number) + + assert isinstance(faildescr, AbstractFailDescr) + + arglocs = self.rebuild_faillocs_from_descr(faildescr, inputargs) + + regalloc = Regalloc(assembler=self) + allgcrefs = [] + operations = regalloc.prepare_bridge(inputargs, arglocs, + operations, + allgcrefs, + self.current_clt.frame_info) + self.reserve_gcref_table(allgcrefs) + startpos = self.mc.get_relative_pos() + + self._check_frame_depth(self.mc, regalloc.get_gcmap()) + + bridgestartpos = self.mc.get_relative_pos() + frame_depth_no_fixed_size = self._assemble(regalloc, inputargs, operations) + + codeendpos = self.mc.get_relative_pos() + + self.write_pending_failure_recoveries() + + fullsize = self.mc.get_relative_pos() + rawstart = self.materialize_loop(original_loop_token) + + self.patch_gcref_table(original_loop_token, rawstart) + self.process_pending_guards(rawstart) + + debug_start("jit-backend-addr") + debug_print("bridge out of Guard 0x%x has address 0x%x to 0x%x" % + (r_uint(descr_number), r_uint(rawstart + startpos), + r_uint(rawstart + codeendpos))) + debug_print(" gc table: 0x%x" % r_uint(rawstart)) + debug_print(" jump target: 0x%x" % r_uint(rawstart + startpos)) + debug_print(" resops: 0x%x" % r_uint(rawstart + bridgestartpos)) + debug_print(" failures: 0x%x" % r_uint(rawstart + codeendpos)) + debug_print(" end: 0x%x" % r_uint(rawstart + fullsize)) + debug_stop("jit-backend-addr") + + # patch the jump from original guard + self.patch_trace(faildescr, original_loop_token, + rawstart + startpos, regalloc) + + self.patch_stack_checks(frame_depth_no_fixed_size + JITFRAME_FIXED_SIZE, + rawstart) + if not we_are_translated(): + if log: + self.mc._dump_trace(rawstart, 'bridge.asm') + + ops_offset = self.mc.ops_offset + frame_depth = max(self.current_clt.frame_info.jfi_frame_depth, + frame_depth_no_fixed_size + JITFRAME_FIXED_SIZE) + self.fixup_target_tokens(rawstart) + self.update_frame_depth(frame_depth) + + if logger: + log = logger.log_trace(jl.MARK_TRACE_ASM, None, self.mc) + log.write(inputargs, operations, ops_offset) + # log that the already written bridge is stitched to a descr! + logger.log_patch_guard(descr_number, rawstart) + + # legacy + if logger.logger_ops: + logger.logger_ops.log_bridge(inputargs, operations, "rewritten", + faildescr, ops_offset=ops_offset) + + self.teardown() + + return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos) + def setup(self, looptoken): BaseAssembler.setup(self, looptoken) assert self.memcpy_addr != 0, 'setup_once() not called?' @@ -220,6 +303,9 @@ def _check_frame_depth_debug(self, mc): pass + def _check_frame_depth(self, mc, gcmap): + pass # XXX + def update_frame_depth(self, frame_depth): baseofs = self.cpu.get_baseofs_of_frame_field() self.current_clt.frame_info.update_frame_depth(baseofs, frame_depth) @@ -264,6 +350,9 @@ gcreftracers.append(tracer) # keepalive self.teardown_gcrefs_list() + def patch_stack_checks(self, framedepth, rawstart): + pass # XXX + def load_from_gc_table(self, regnum, index): address_in_buffer = index * WORD # at the start of the buffer p_location = self.mc.get_relative_pos(break_basic_block=False) @@ -281,6 +370,14 @@ # self.codemap.get_final_bytecode(res, size)) return res + def patch_trace(self, faildescr, looptoken, bridge_addr, regalloc): + b = InstrBuilder() + patch_addr = faildescr.adr_jump_offset + assert patch_addr != 0 + b.BL(bridge_addr) + b.copy_to_raw_memory(patch_addr) + faildescr.adr_jump_offset = 0 + def process_pending_guards(self, block_start): clt = self.current_clt for tok in self.pending_guards: diff --git a/rpython/jit/backend/aarch64/codebuilder.py b/rpython/jit/backend/aarch64/codebuilder.py --- a/rpython/jit/backend/aarch64/codebuilder.py +++ b/rpython/jit/backend/aarch64/codebuilder.py @@ -137,7 +137,7 @@ def BL(self, target): target = rffi.cast(lltype.Signed, target) - self.gen_load_int(r.ip0.value, target) + self.gen_load_int_full(r.ip0.value, target) self.BR(r.ip0.value) def BR(self, reg): @@ -147,6 +147,12 @@ def BRK(self): self.write32(0b11010100001 << 21) + def gen_load_int_full(self, r, value): + self.MOVZ_r_u16(r, value & 0xFFFF, 0) + self.MOVK_r_u16(r, (value >> 16) & 0xFFFF, 16) + self.MOVK_r_u16(r, (value >> 32) & 0xFFFF, 32) + self.MOVK_r_u16(r, (value >> 48) & 0xFFFF, 48) + def gen_load_int(self, r, value): """r is the register number, value is the value to be loaded to the register""" diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -120,7 +120,7 @@ if target_token in self.target_tokens_currently_compiling: self.mc.B_ofs(target) else: - self.mc.B(target) + self.mc.BL(target) def emit_op_finish(self, op, arglocs): base_ofs = self.cpu.get_baseofs_of_frame_field() diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -1,6 +1,7 @@ from rpython.jit.backend.aarch64 import registers as r from rpython.jit.backend.aarch64 import locations +from rpython.jit.backend.aarch64.arch import WORD, JITFRAME_FIXED_SIZE from rpython.jit.metainterp.history import (Const, ConstInt, ConstFloat, ConstPtr, @@ -14,6 +15,8 @@ from rpython.jit.backend.aarch64 import registers as r from rpython.jit.backend.arm.jump import remap_frame_layout_mixed from rpython.jit.backend.aarch64.locations import imm +from rpython.jit.backend.llsupport.gcmap import allocate_gcmap + class TempInt(TempVar): @@ -450,6 +453,64 @@ self.rm._check_invariants() self.vfprm._check_invariants() + def prepare_bridge(self, inputargs, arglocs, operations, allgcrefs, + frame_info): + operations = self._prepare(inputargs, operations, allgcrefs) + self._update_bindings(arglocs, inputargs) + return operations + + def _update_bindings(self, locs, inputargs): + used = {} + i = 0 + for loc in locs: + if loc is None: + loc = r.fp + arg = inputargs[i] + i += 1 + if loc.is_core_reg(): + self.rm.reg_bindings[arg] = loc + used[loc] = None + elif loc.is_vfp_reg(): + self.vfprm.reg_bindings[arg] = loc + used[loc] = None + else: + assert loc.is_stack() + self.frame_manager.bind(arg, loc) + + # XXX combine with x86 code and move to llsupport + self.rm.free_regs = [] + for reg in self.rm.all_regs: + if reg not in used: + self.rm.free_regs.append(reg) + self.vfprm.free_regs = [] + for reg in self.vfprm.all_regs: + if reg not in used: + self.vfprm.free_regs.append(reg) + # note: we need to make a copy of inputargs because possibly_free_vars + # is also used on op args, which is a non-resizable list + self.possibly_free_vars(list(inputargs)) + self.fm.finish_binding() + self._check_invariants() + + def get_gcmap(self, forbidden_regs=[], noregs=False): + frame_depth = self.fm.get_frame_depth() + gcmap = allocate_gcmap(self.assembler, + frame_depth, JITFRAME_FIXED_SIZE) + for box, loc in self.rm.reg_bindings.iteritems(): + if loc in forbidden_regs: + continue + if box.type == REF and self.rm.is_still_alive(box): + assert not noregs + assert loc.is_core_reg() + val = loc.value + gcmap[val // WORD // 8] |= r_uint(1) << (val % (WORD * 8)) + for box, loc in self.fm.bindings.iteritems(): + if box.type == REF and self.rm.is_still_alive(box): + assert loc.is_stack() + val = loc.position + JITFRAME_FIXED_SIZE + gcmap[val // WORD // 8] |= r_uint(1) << (val % (WORD * 8)) + return gcmap + def get_final_frame_depth(self): return self.frame_manager.get_frame_depth() diff --git a/rpython/jit/backend/aarch64/runner.py b/rpython/jit/backend/aarch64/runner.py --- a/rpython/jit/backend/aarch64/runner.py +++ b/rpython/jit/backend/aarch64/runner.py @@ -2,6 +2,7 @@ from rpython.rtyper.lltypesystem import llmemory, lltype from rpython.jit.backend.aarch64.assembler import AssemblerARM64 from rpython.jit.backend.aarch64 import registers as r +from rpython.jit.backend.aarch64.regalloc import VFPRegisterManager from rpython.jit.backend.llsupport.llmodel import AbstractLLCPU class CPU_ARM64(AbstractLLCPU): @@ -9,6 +10,9 @@ backend_name = "aarch64" frame_reg = r.fp all_reg_indexes = range(len(r.all_regs)) + gen_regs = r.all_regs + float_regs = VFPRegisterManager.all_regs + IS_64_BIT = True @@ -23,6 +27,14 @@ def setup_once(self): self.assembler.setup_once() + def compile_bridge(self, faildescr, inputargs, operations, + original_loop_token, log=True, logger=None): + clt = original_loop_token.compiled_loop_token + clt.compiling_a_bridge() + return self.assembler.assemble_bridge(logger, faildescr, inputargs, + operations, + original_loop_token, log=log) + def cast_ptr_to_int(x): adr = llmemory.cast_ptr_to_adr(x) return CPU_ARM64.cast_adr_to_int(adr) From pypy.commits at gmail.com Wed Mar 6 10:05:17 2019 From: pypy.commits at gmail.com (fijal) Date: Wed, 06 Mar 2019 07:05:17 -0800 (PST) Subject: [pypy-commit] pypy arm64: in progress Message-ID: <5c7fe1ad.1c69fb81.affd2.118b@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96220:d301852919b9 Date: 2019-03-06 15:04 +0000 http://bitbucket.org/pypy/pypy/changeset/d301852919b9/ Log: in progress diff --git a/rpython/jit/backend/aarch64/codebuilder.py b/rpython/jit/backend/aarch64/codebuilder.py --- a/rpython/jit/backend/aarch64/codebuilder.py +++ b/rpython/jit/backend/aarch64/codebuilder.py @@ -114,6 +114,11 @@ base = 0b11101011000 self.write32((base << 21) | (rm << 16) | (rn << 5) | 0b11111) + def CMP_ri(self, rn, imm): + base = 0b1111000100 + assert 0 <= imm <= 4095 + self.write32((base << 22) | (imm << 10) | (rn << 5) | 0b11111) + def B_ofs(self, ofs): base = 0b000101 assert ofs & 0x3 == 0 diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -110,6 +110,20 @@ def emit_guard_op_guard_true(self, guard_op, fcond, arglocs): self._emit_guard(guard_op, fcond, arglocs) + def emit_guard_op_guard_false(self, guard_op, fcond, arglocs): + self._emit_guard(guard_op, c.get_opposite_of(fcond), arglocs) + + def load_condition_into_cc(self, loc): + if not loc.is_core_reg(): + assert loc.is_stack() + self.regalloc_mov(loc, r.ip0) + loc = r.ip0 + self.mc.CMP_ri(loc.value, 0) + + def emit_op_guard_false(self, op, arglocs): + self.load_condition_into_cc(arglocs[1]) + self._emit_guard(op, c.NE, arglocs) + def emit_op_label(self, op, arglocs): pass diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -392,17 +392,25 @@ locs = [] return locs - def prepare_guard_op_guard_true(self, op, prevop): + def guard_impl(self, op, prevop): fcond = self.assembler.dispatch_comparison(prevop) # result is in CC + return self._guard_impl(op), fcond + def _guard_impl(self, op): arglocs = [None] * (len(op.getfailargs()) + 1) arglocs[0] = imm(self.frame_manager.get_frame_depth()) failargs = op.getfailargs() for i in range(len(failargs)): if failargs[i]: arglocs[i + 1] = self.loc(failargs[i]) - return arglocs, fcond + return arglocs + + prepare_guard_op_guard_true = guard_impl + prepare_guard_op_guard_false = guard_impl + + prepare_op_guard_true = _guard_impl + prepare_op_guard_false = _guard_impl prepare_op_nursery_ptr_increment = prepare_op_int_add From pypy.commits at gmail.com Wed Mar 6 11:39:32 2019 From: pypy.commits at gmail.com (antocuni) Date: Wed, 06 Mar 2019 08:39:32 -0800 (PST) Subject: [pypy-commit] pypy default: update the docs about GC hooks: the unit of the duration fields was changed by the expose-gc-time branch, but the docs were not updated accordingly :(. While at it, document the various functions __pypy__.debug_*, and move the note about debug_get_timestamp_unit() into the proper place Message-ID: <5c7ff7c4.1c69fb81.e9dd5.5996@mx.google.com> Author: Antonio Cuni Branch: Changeset: r96221:c42a88cbac10 Date: 2019-03-06 17:35 +0100 http://bitbucket.org/pypy/pypy/changeset/c42a88cbac10/ Log: update the docs about GC hooks: the unit of the duration fields was changed by the expose-gc-time branch, but the docs were not updated accordingly :(. While at it, document the various functions __pypy__.debug_*, and move the note about debug_get_timestamp_unit() into the proper place diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst --- a/pypy/doc/__pypy__-module.rst +++ b/pypy/doc/__pypy__-module.rst @@ -32,13 +32,6 @@ internally the same object. - ``set_debug`` - - ``debug_start`` - - ``debug_stop`` - - ``debug_print`` - - ``debug_print_once`` - - ``debug_flush`` - - ``debug_read_timestamp`` - - ``debug_get_timestamp_unit`` - ``builtinify(func)``: To implement at app-level modules that are, in CPython, implemented in C: this decorator protects a function from being ever bound @@ -135,6 +128,59 @@ - ``builders.StringBuilder`` - ``builders.UnicodeBuilder`` +Interacting with the PyPy debug log +------------------------------------ + +The following functions can be used to write your own content to the +:ref:`PYPYLOG `. + + - ``debug_start(category, timestamp=False)``: open a new section; if + ``timestamp`` is ``True``, also return the timestamp which was written to + the log. + + - ``debug_stop(category, timestamp=False)``: close a section opened by + ``debug_start``. + + - ``debug_print(...)``: print arbitrary text to the log. + + - ``debug_print_once(category, ...)``: equivalent to ``debug_start`` + + ``debug_print`` + ``debug_stop``. + + - ``debug_flush``: flush the log. + + - ``debug_read_timestamp()``: read the timestamp from the same timer used by + the log. + + - ``debug_get_timestamp_unit()``: get the unit of the value returned by + ``debug_read_timestamp()``. + + +Depending on the architecture and operating system, PyPy uses different ways +to read timestamps, so the timestamps used in the log file are expressed in +varying units. It is possible to know which by calling +``debug_get_timestamp_unit()``, which can be one of the following values: + +``tsc`` + The default on ``x86`` machines: timestamps are expressed in CPU ticks, as + read by the `Time Stamp Counter`_. + +``ns`` + Timestamps are expressed in nanoseconds. + +``QueryPerformanceCounter`` + On Windows, in case for some reason ``tsc`` is not available: timestamps + are read using the win API ``QueryPerformanceCounter()``. + + +Unfortunately, there does not seem to be a reliable standard way for +converting ``tsc`` ticks into nanoseconds, although in practice on modern CPUs +it is enough to divide the ticks by the maximum nominal frequency of the CPU. +For this reason, PyPy gives the raw value, and leaves the job of doing the +conversion to external libraries. + +.. _`Time Stamp Counter`: https://en.wikipedia.org/wiki/Time_Stamp_Counter + + Transparent Proxy Functionality ------------------------------- diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -203,7 +203,7 @@ ``duration`` The total time spent inside minor collections since the last hook - call. See below for more information on the unit. + call, in seconds. ``duration_min`` The duration of the fastest minor collection since the last hook call. @@ -265,30 +265,6 @@ ``gc-collect-done`` is used only to give additional stats, but doesn't do any actual work. -A note about the ``duration`` field: depending on the architecture and -operating system, PyPy uses different ways to read timestamps, so ``duration`` -is expressed in varying units. It is possible to know which by calling -``__pypy__.debug_get_timestamp_unit()``, which can be one of the following -values: - -``tsc`` - The default on ``x86`` machines: timestamps are expressed in CPU ticks, as - read by the `Time Stamp Counter`_. - -``ns`` - Timestamps are expressed in nanoseconds. - -``QueryPerformanceCounter`` - On Windows, in case for some reason ``tsc`` is not available: timestamps - are read using the win API ``QueryPerformanceCounter()``. - - -Unfortunately, there does not seem to be a reliable standard way for -converting ``tsc`` ticks into nanoseconds, although in practice on modern CPUs -it is enough to divide the ticks by the maximum nominal frequency of the CPU. -For this reason, PyPy gives the raw value, and leaves the job of doing the -conversion to external libraries. - Here is an example of GC hooks in use:: import sys @@ -321,8 +297,6 @@ lst = [lst, 1, 2, 3] -.. _`Time Stamp Counter`: https://en.wikipedia.org/wiki/Time_Stamp_Counter - .. _minimark-environment-variables: Environment variables diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst --- a/pypy/doc/man/pypy.1.rst +++ b/pypy/doc/man/pypy.1.rst @@ -99,6 +99,8 @@ If set, equivalent to the ``-W`` option (warning control). The value should be a comma-separated list of ``-W`` parameters. +.. _pypylog: + ``PYPYLOG`` If set to a non-empty value, enable logging, the format is: From pypy.commits at gmail.com Wed Mar 6 12:32:24 2019 From: pypy.commits at gmail.com (arigo) Date: Wed, 06 Mar 2019 09:32:24 -0800 (PST) Subject: [pypy-commit] pypy default: update to cffi/6d435908617d Message-ID: <5c800428.1c69fb81.19124.7598@mx.google.com> Author: Armin Rigo Branch: Changeset: r96222:ddbc46f09114 Date: 2019-03-06 18:32 +0100 http://bitbucket.org/pypy/pypy/changeset/ddbc46f09114/ Log: update to cffi/6d435908617d diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.12.1", ("This test_c.py file is for testing a version" +assert __version__ == "1.12.2", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): From pypy.commits at gmail.com Wed Mar 6 12:33:56 2019 From: pypy.commits at gmail.com (arigo) Date: Wed, 06 Mar 2019 09:33:56 -0800 (PST) Subject: [pypy-commit] pypy default: update to cffi/6d435908617d (really this time) Message-ID: <5c800484.1c69fb81.65ca4.8d61@mx.google.com> Author: Armin Rigo Branch: Changeset: r96223:dc08ce14d28b Date: 2019-03-06 18:34 +0100 http://bitbucket.org/pypy/pypy/changeset/dc08ce14d28b/ Log: update to cffi/6d435908617d (really this time) diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -5,8 +5,8 @@ from .error import CDefError, FFIError, VerificationError, VerificationMissing from .error import PkgConfigError -__version__ = "1.12.1" -__version_info__ = (1, 12, 1) +__version__ = "1.12.2" +__version_info__ = (1, 12, 2) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -221,7 +221,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.12.1" + "\ncompiled with cffi version: 1.12.2" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi from rpython.rtyper.lltypesystem import rffi -VERSION = "1.12.1" +VERSION = "1.12.2" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: From pypy.commits at gmail.com Wed Mar 6 12:34:48 2019 From: pypy.commits at gmail.com (arigo) Date: Wed, 06 Mar 2019 09:34:48 -0800 (PST) Subject: [pypy-commit] pypy default: update to cffi/6d435908617d (3rd try) Message-ID: <5c8004b8.1c69fb81.77353.c74a@mx.google.com> Author: Armin Rigo Branch: Changeset: r96224:28c9c798ba05 Date: 2019-03-06 18:35 +0100 http://bitbucket.org/pypy/pypy/changeset/28c9c798ba05/ Log: update to cffi/6d435908617d (3rd try) diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.12.1 +Version: 1.12.2 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski From pypy.commits at gmail.com Wed Mar 6 15:56:12 2019 From: pypy.commits at gmail.com (mattip) Date: Wed, 06 Mar 2019 12:56:12 -0800 (PST) Subject: [pypy-commit] buildbot default: add temporary 32bit buildbot, hopefully bencher4 will return Message-ID: <5c8033ec.1c69fb81.cab74.6b4a@mx.google.com> Author: Matti Picus Branch: Changeset: r1074:4cc524e68457 Date: 2019-03-06 22:39 +0200 http://bitbucket.org/pypy/buildbot/changeset/4cc524e68457/ Log: add temporary 32bit buildbot, hopefully bencher4 will return diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -418,14 +418,14 @@ 'builders': [ {"name": LINUX32OWN, - "slavenames": ["bencher4_32"], + "slavenames": ["bencher4_32", "salsa_32"], "builddir": LINUX32OWN, "factory": pypyOwnTestFactory, "category": 'linux32', "locks": [TannitCPU.access('counting')], }, {"name": LINUX32RPYTHON, - "slavenames": ["bencher4_32"], + "slavenames": ["bencher4_32", "salsa_32"], "builddir": LINUX32RPYTHON, "factory": pypyRPythonTestFactory, "category": 'linux32', @@ -481,7 +481,7 @@ }, {"name" : JITLINUX32, #"slavenames": ["allegro32"], - "slavenames": ["bencher4_32"], + "slavenames": ["bencher4_32", "salsa_32"], 'builddir' : JITLINUX32, 'factory' : pypyJITTranslatedTestFactory, 'category' : 'linux32', From pypy.commits at gmail.com Fri Mar 8 09:00:42 2019 From: pypy.commits at gmail.com (stevie_92) Date: Fri, 08 Mar 2019 06:00:42 -0800 (PST) Subject: [pypy-commit] pypy cpyext-gc-cycle: Implemented garbage_pypy and improved tests Message-ID: <5c82758a.1c69fb81.7f23a.244d@mx.google.com> Author: Stefan Beyer Branch: cpyext-gc-cycle Changeset: r96225:5d1c0b97ef5a Date: 2019-03-08 14:23 +0100 http://bitbucket.org/pypy/pypy/changeset/5d1c0b97ef5a/ Log: Implemented garbage_pypy and improved tests diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -3141,6 +3141,7 @@ self.rrc_pyobj_isolate_list = self._rrc_gc_list_new() self.rrc_pyobj_dead_list = self._rrc_gc_list_new() self.rrc_pyobj_garbage_list = self._rrc_gc_list_new() + self.rrc_garbage_to_trace = self.AddressStack() self.rrc_gc_as_pyobj = gc_as_pyobj self.rrc_pyobj_as_gc = pyobj_as_gc self.rrc_finalizer_type = finalizer_type @@ -3246,15 +3247,14 @@ self.rrc_state = self.RAWREFCOUNT_STATE_DEFAULT def rawrefcount_next_garbage_pypy(self): - # We assume that next_garbage_pypy is always called before - # next_garbage_pyobj. As pypy objects can only be in garbage, if there - # is at least one pyobj in garbage, we can use this optimization. - if self._rrc_gc_list_is_empty(self.rrc_pyobj_garbage_list): - return lltype.nullptr(llmemory.GCREF.TO) + if self.rrc_garbage_to_trace.non_empty(): + # remove one object from the wavefront and move the wavefront + obj = self.rrc_garbage_to_trace.pop() + if self._rrc_garbage_visit(obj): + return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) + else: + return lltype.nullptr(llmemory.GCREF.TO) else: - # TODO: return the next pypy object which is marked with GCFLAG_GARBAGE and - # remove the flag from this object. We can safely assume that objects - # do not move, as this can only happen to old objects. return lltype.nullptr(llmemory.GCREF.TO) def rawrefcount_next_garbage_pyobj(self): @@ -3620,9 +3620,28 @@ if pyobj.c_ob_pypy_link <> 0: intobj = pyobj.c_ob_pypy_link obj = llmemory.cast_int_to_adr(intobj) + self.rrc_garbage_to_trace.append(obj) self.objects_to_trace.append(obj) self.visit_all_objects() + def _rrc_collect_obj(self, obj, ignored): + llop.debug_nonnull_pointer(lltype.Void, obj) + self.rrc_garbage_to_trace.append(obj) + _rrc_collect_obj._always_inline_ = True + + def _rrc_collect_ref_rec(self, root, ignored): + self._rrc_collect_obj(root.address[0], None) + + def _rrc_garbage_visit(self, obj): + # If GCFLAG_GARBAGE is set, remove the flag and trace the object + hdr = self.header(obj) + if not (hdr.tid & GCFLAG_GARBAGE): + return False + hdr.tid &= ~GCFLAG_GARBAGE + if self.has_gcptr(llop.extract_ushort(llgroup.HALFWORD, hdr.tid)): + self.trace(obj, self._rrc_collect_ref_rec, None) + return True + def _rrc_check_finalizer(self): # Check, if the cyclic isolate from the last collection cycle # is reachable from outside, after the finalizers have been diff --git a/rpython/memory/gc/test/dot/garbage_cross_simple_1.dot b/rpython/memory/gc/test/dot/garbage_cross_simple_1.dot --- a/rpython/memory/gc/test/dot/garbage_cross_simple_1.dot +++ b/rpython/memory/gc/test/dot/garbage_cross_simple_1.dot @@ -3,8 +3,12 @@ "b" [type=B, alive=y, garbage=y]; "c" [type=P, alive=y, garbage=y]; "d" [type=P, alive=y, rooted=y]; + "e" [type=C, alive=y, garbage=y]; + "f" [type=C, alive=y, ext_refcnt=1]; "a" -> "b"; "b" -> "a"; "b" -> "c"; "c" -> "d"; + "a" -> "e"; + "a" -> "f"; } \ No newline at end of file diff --git a/rpython/memory/gc/test/test_rawrefcount.py b/rpython/memory/gc/test/test_rawrefcount.py --- a/rpython/memory/gc/test/test_rawrefcount.py +++ b/rpython/memory/gc/test/test_rawrefcount.py @@ -672,10 +672,15 @@ # have been added to the garbage list for name in nodes: n = nodes[name] + garbage = n.info.garbage if n.info.alive: if n.info.type == "C": - assert n.info.garbage != (n.raddr not in garbage_pyobj) + assert garbage != (n.raddr not in garbage_pyobj), \ + "PyObject should " + ("" if garbage else "not ") + \ + "be in garbage" else: - assert n.info.garbage != (n.pref not in garbage_pypy) + assert garbage != (n.pref not in garbage_pypy), \ + "Object should " + ("" if garbage else "not ") + \ + "be in garbage" else: - assert not n.info.garbage \ No newline at end of file + assert not garbage, "Object is dead, but should be in garbage" From pypy.commits at gmail.com Sat Mar 9 13:42:49 2019 From: pypy.commits at gmail.com (mattip) Date: Sat, 09 Mar 2019 10:42:49 -0800 (PST) Subject: [pypy-commit] pypy default: change test of utf8 length, len(u) != codepoints on narrow builds Message-ID: <5c840929.1c69fb81.d0633.4ba4@mx.google.com> Author: Matti Picus Branch: Changeset: r96228:ead9c5bf9d9c Date: 2019-03-09 20:43 +0200 http://bitbucket.org/pypy/pypy/changeset/ead9c5bf9d9c/ Log: change test of utf8 length, len(u) != codepoints on narrow builds diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -44,7 +44,7 @@ if not we_are_translated(): try: # best effort, too expensive to handle surrogates - ulength = len(utf8str.decode('utf8')) + ulength = rutf8.codepoints_in_utf(utf8str) except: ulength = length assert ulength == length From pypy.commits at gmail.com Sat Mar 9 13:42:51 2019 From: pypy.commits at gmail.com (mattip) Date: Sat, 09 Mar 2019 10:42:51 -0800 (PST) Subject: [pypy-commit] pypy py3.6: merge default into branch Message-ID: <5c84092b.1c69fb81.5d06a.b183@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r96229:d4d3c9ef7fcf Date: 2019-03-09 20:44 +0200 http://bitbucket.org/pypy/pypy/changeset/d4d3c9ef7fcf/ Log: merge default into branch diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -41,7 +41,7 @@ if not we_are_translated(): try: # best effort, too expensive to handle surrogates - ulength = len(utf8str.decode('utf8')) + ulength = rutf8.codepoints_in_utf(utf8str) except: ulength = length assert ulength == length From pypy.commits at gmail.com Sat Mar 9 14:49:45 2019 From: pypy.commits at gmail.com (mattip) Date: Sat, 09 Mar 2019 11:49:45 -0800 (PST) Subject: [pypy-commit] buildbot default: add locale setup for debian Message-ID: <5c8418d9.1c69fb81.b3e9f.615e@mx.google.com> Author: Matti Picus Branch: Changeset: r1075:4c86f3a702e9 Date: 2019-03-09 21:47 +0200 http://bitbucket.org/pypy/buildbot/changeset/4c86f3a702e9/ Log: add locale setup for debian diff --git a/README-CHROOT b/README-CHROOT --- a/README-CHROOT +++ b/README-CHROOT @@ -50,7 +50,11 @@ chmod buildslave.buildslave -R /extra1/stretch32/home/buildslave apt install virtualenv buildbot-slave python-pytest python-hypothesis \ -netbase gdb ncurses-term pypy +netbase gdb ncurses-term pypy locales and more from the instructions on https://bitbucket.org/pypy/buildbot +also, to make the rlocale tests work + +dpkg-reconfigure locales +# choose to generate all encodings, choose en_US.UTF-8 From pypy.commits at gmail.com Sat Mar 9 14:54:57 2019 From: pypy.commits at gmail.com (mattip) Date: Sat, 09 Mar 2019 11:54:57 -0800 (PST) Subject: [pypy-commit] pypy newmemoryview-app-level: merge default into branch Message-ID: <5c841a11.1c69fb81.bd341.16a2@mx.google.com> Author: Matti Picus Branch: newmemoryview-app-level Changeset: r96230:47a045ff6be8 Date: 2019-03-09 21:54 +0200 http://bitbucket.org/pypy/pypy/changeset/47a045ff6be8/ Log: merge default into branch diff --git a/extra_tests/requirements.txt b/extra_tests/requirements.txt --- a/extra_tests/requirements.txt +++ b/extra_tests/requirements.txt @@ -1,3 +1,3 @@ -pytest +pytest<=4.0 hypothesis vmprof diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.12.1 +Version: 1.12.2 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -5,8 +5,8 @@ from .error import CDefError, FFIError, VerificationError, VerificationMissing from .error import PkgConfigError -__version__ = "1.12.1" -__version_info__ = (1, 12, 1) +__version__ = "1.12.2" +__version_info__ = (1, 12, 2) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -221,7 +221,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.12.1" + "\ncompiled with cffi version: 1.12.2" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst --- a/pypy/doc/__pypy__-module.rst +++ b/pypy/doc/__pypy__-module.rst @@ -13,7 +13,6 @@ ``if platform.python_implementation == 'PyPy'`` block or otherwise hidden from the CPython interpreter. - Generally available functionality --------------------------------- @@ -29,17 +28,10 @@ create a `memoryview` instance with the data from ``buffer`` and the specified itemsize, format, and optional shape and strides. - - ``bufferable``: a base class that must override the - ``__buffer__(self, flags)`` method. This method should return a memoryview + - ``bufferable``: a base class that provides a ``__buffer__(self, flags)`` + method for subclasses to override. This method should return a memoryview instance of the class instance. It is called by the C-API's ``tp_as_buffer. bf_getbuffer``. - - - ``debug_stop`` - - ``debug_print`` - - ``debug_print_once`` - - ``debug_flush`` - - ``debug_read_timestamp`` - - ``debug_get_timestamp_unit`` - ``builtinify(func)``: To implement at app-level modules that are, in CPython, implemented in C: this decorator protects a function from being ever bound @@ -136,6 +128,59 @@ - ``builders.StringBuilder`` - ``builders.UnicodeBuilder`` +Interacting with the PyPy debug log +------------------------------------ + +The following functions can be used to write your own content to the +:ref:`PYPYLOG `. + + - ``debug_start(category, timestamp=False)``: open a new section; if + ``timestamp`` is ``True``, also return the timestamp which was written to + the log. + + - ``debug_stop(category, timestamp=False)``: close a section opened by + ``debug_start``. + + - ``debug_print(...)``: print arbitrary text to the log. + + - ``debug_print_once(category, ...)``: equivalent to ``debug_start`` + + ``debug_print`` + ``debug_stop``. + + - ``debug_flush``: flush the log. + + - ``debug_read_timestamp()``: read the timestamp from the same timer used by + the log. + + - ``debug_get_timestamp_unit()``: get the unit of the value returned by + ``debug_read_timestamp()``. + + +Depending on the architecture and operating system, PyPy uses different ways +to read timestamps, so the timestamps used in the log file are expressed in +varying units. It is possible to know which by calling +``debug_get_timestamp_unit()``, which can be one of the following values: + +``tsc`` + The default on ``x86`` machines: timestamps are expressed in CPU ticks, as + read by the `Time Stamp Counter`_. + +``ns`` + Timestamps are expressed in nanoseconds. + +``QueryPerformanceCounter`` + On Windows, in case for some reason ``tsc`` is not available: timestamps + are read using the win API ``QueryPerformanceCounter()``. + + +Unfortunately, there does not seem to be a reliable standard way for +converting ``tsc`` ticks into nanoseconds, although in practice on modern CPUs +it is enough to divide the ticks by the maximum nominal frequency of the CPU. +For this reason, PyPy gives the raw value, and leaves the job of doing the +conversion to external libraries. + +.. _`Time Stamp Counter`: https://en.wikipedia.org/wiki/Time_Stamp_Counter + + Transparent Proxy Functionality ------------------------------- diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -203,7 +203,7 @@ ``duration`` The total time spent inside minor collections since the last hook - call. See below for more information on the unit. + call, in seconds. ``duration_min`` The duration of the fastest minor collection since the last hook call. @@ -265,30 +265,6 @@ ``gc-collect-done`` is used only to give additional stats, but doesn't do any actual work. -A note about the ``duration`` field: depending on the architecture and -operating system, PyPy uses different ways to read timestamps, so ``duration`` -is expressed in varying units. It is possible to know which by calling -``__pypy__.debug_get_timestamp_unit()``, which can be one of the following -values: - -``tsc`` - The default on ``x86`` machines: timestamps are expressed in CPU ticks, as - read by the `Time Stamp Counter`_. - -``ns`` - Timestamps are expressed in nanoseconds. - -``QueryPerformanceCounter`` - On Windows, in case for some reason ``tsc`` is not available: timestamps - are read using the win API ``QueryPerformanceCounter()``. - - -Unfortunately, there does not seem to be a reliable standard way for -converting ``tsc`` ticks into nanoseconds, although in practice on modern CPUs -it is enough to divide the ticks by the maximum nominal frequency of the CPU. -For this reason, PyPy gives the raw value, and leaves the job of doing the -conversion to external libraries. - Here is an example of GC hooks in use:: import sys @@ -321,8 +297,6 @@ lst = [lst, 1, 2, 3] -.. _`Time Stamp Counter`: https://en.wikipedia.org/wiki/Time_Stamp_Counter - .. _minimark-environment-variables: Environment variables diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst --- a/pypy/doc/man/pypy.1.rst +++ b/pypy/doc/man/pypy.1.rst @@ -99,6 +99,8 @@ If set, equivalent to the ``-W`` option (warning control). The value should be a comma-separated list of ``-W`` parameters. +.. _pypylog: + ``PYPYLOG`` If set to a non-empty value, enable logging, the format is: diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi from rpython.rtyper.lltypesystem import rffi -VERSION = "1.12.1" +VERSION = "1.12.2" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.12.1", ("This test_c.py file is for testing a version" +assert __version__ == "1.12.2", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -44,7 +44,7 @@ if not we_are_translated(): try: # best effort, too expensive to handle surrogates - ulength = len(utf8str.decode('utf8')) + ulength = rutf8.codepoints_in_utf(utf8str) except: ulength = length assert ulength == length From pypy.commits at gmail.com Sun Mar 10 06:42:47 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 03:42:47 -0700 (PDT) Subject: [pypy-commit] pypy default: add needed module for tests Message-ID: <5c84ea27.1c69fb81.2f906.316e@mx.google.com> Author: Matti Picus Branch: Changeset: r96231:b73eb7fdb240 Date: 2019-03-10 12:41 +0200 http://bitbucket.org/pypy/pypy/changeset/b73eb7fdb240/ Log: add needed module for tests diff --git a/pypy/module/_multiprocessing/test/test_memory.py b/pypy/module/_multiprocessing/test/test_memory.py --- a/pypy/module/_multiprocessing/test/test_memory.py +++ b/pypy/module/_multiprocessing/test/test_memory.py @@ -5,7 +5,9 @@ '_rawffi', 'itertools', 'signal', 'select', 'binascii')) - if sys.platform != 'win32': + if sys.platform == 'win32': + spaceconfig['usemodules'] += ('_cffi_backend',) + else: spaceconfig['usemodules'] += ('fcntl',) def test_address_of(self): diff --git a/pypy/module/_multiprocessing/test/test_semaphore.py b/pypy/module/_multiprocessing/test/test_semaphore.py --- a/pypy/module/_multiprocessing/test/test_semaphore.py +++ b/pypy/module/_multiprocessing/test/test_semaphore.py @@ -10,7 +10,7 @@ 'binascii', 'struct')) if sys.platform == 'win32': - spaceconfig['usemodules'] += ('_rawffi',) + spaceconfig['usemodules'] += ('_rawffi', '_cffi_backend') else: spaceconfig['usemodules'] += ('fcntl',) diff --git a/pypy/module/_multiprocessing/test/test_win32.py b/pypy/module/_multiprocessing/test/test_win32.py --- a/pypy/module/_multiprocessing/test/test_win32.py +++ b/pypy/module/_multiprocessing/test/test_win32.py @@ -2,7 +2,7 @@ import sys class AppTestWin32: - spaceconfig = dict(usemodules=('_multiprocessing', + spaceconfig = dict(usemodules=('_multiprocessing', _cffi_backend', 'signal', '_rawffi', 'binascii')) def setup_class(cls): From pypy.commits at gmail.com Sun Mar 10 06:42:49 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 03:42:49 -0700 (PDT) Subject: [pypy-commit] pypy default: merge newmemoryview-app-level into default Message-ID: <5c84ea29.1c69fb81.b213d.adaa@mx.google.com> Author: Matti Picus Branch: Changeset: r96232:750f4840080b Date: 2019-03-10 12:42 +0200 http://bitbucket.org/pypy/pypy/changeset/750f4840080b/ Log: merge newmemoryview-app-level into default diff --git a/extra_tests/ctypes_tests/test_structures.py b/extra_tests/ctypes_tests/test_structures.py --- a/extra_tests/ctypes_tests/test_structures.py +++ b/extra_tests/ctypes_tests/test_structures.py @@ -124,12 +124,15 @@ ms.n = 0xff00 return repr(ba[:]) + nstruct = dostruct(Native) if sys.byteorder == 'little': - assert dostruct(Native) == dostruct(Little) - assert dostruct(Native) != dostruct(Big) + assert nstruct == dostruct(Little) + assert nstruct != dostruct(Big) + assert Big._fields_[0][1] is not i else: - assert dostruct(Native) == dostruct(Big) - assert dostruct(Native) != dostruct(Little) + assert nstruct == dostruct(Big) + assert nstruct != dostruct(Little) + assert Little._fields_[0][1] is not i def test_from_buffer_copy(): from array import array @@ -190,3 +193,20 @@ assert sizeof(s) == 3 * sizeof(c_int) assert s.a == 4 # 256 + 4 assert s.b == -123 + +def test_memoryview(): + class S(Structure): + _fields_ = [('a', c_int16), + ('b', c_int16), + ] + + S3 = S * 3 + c_array = (2 * S3)( + S3(S(a=0, b=1), S(a=2, b=3), S(a=4, b=5)), + S3(S(a=6, b=7), S(a=8, b=9), S(a=10, b=11)), + ) + + mv = memoryview(c_array) + assert mv.format == 'T{'} +swappedorder = {'little': '>', 'big': '<'} + +def get_format_str(typ): + if hasattr(typ, '_fields_'): + if hasattr(typ, '_swappedbytes_'): + bo = swappedorder[sys.byteorder] + else: + bo = byteorder[sys.byteorder] + flds = [] + for name, obj in typ._fields_: + # Trim off the leading '<' or '>' + ch = get_format_str(obj)[1:] + if (ch) == 'B': + flds.append(byteorder[sys.byteorder]) + else: + flds.append(bo) + flds.append(ch) + flds.append(':') + flds.append(name) + flds.append(':') + return 'T{' + ''.join(flds) + '}' + elif hasattr(typ, '_type_'): + ch = typ._type_ + return byteorder[sys.byteorder] + ch + else: + raise ValueError('cannot get format string for %r' % typ) diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py --- a/lib_pypy/_ctypes/basics.py +++ b/lib_pypy/_ctypes/basics.py @@ -2,8 +2,15 @@ from _rawffi import alt as _ffi import sys -try: from __pypy__ import builtinify -except ImportError: builtinify = lambda f: f +try: + from __pypy__ import builtinify +except ImportError: + builtinify = lambda f: f + +try: + from __pypy__.bufferable import bufferable +except ImportError: + bufferable = object keepalive_key = str # XXX fix this when provided with test @@ -64,7 +71,7 @@ 'resbuffer' is a _rawffi array of length 1 containing the value, and this returns a general Python object that corresponds. """ - res = object.__new__(self) + res = bufferable.__new__(self) res.__class__ = self res.__dict__['_buffer'] = resbuffer if base is not None: @@ -158,7 +165,7 @@ def __ne__(self, other): return self._obj != other -class _CData(object): +class _CData(bufferable): """ The most basic object for all ctypes types """ __metaclass__ = _CDataMeta diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py --- a/lib_pypy/_ctypes/pointer.py +++ b/lib_pypy/_ctypes/pointer.py @@ -7,8 +7,7 @@ from _ctypes.array import Array, array_get_slice_params, array_slice_getitem,\ array_slice_setitem -try: from __pypy__ import builtinify -except ImportError: builtinify = lambda f: f +from __pypy__ import builtinify, newmemoryview # This cache maps types to pointers to them. _pointer_type_cache = {} @@ -135,6 +134,9 @@ def _as_ffi_pointer_(self, ffitype): return as_ffi_pointer(self, ffitype) + def __buffer__(self, flags): + mv = memoryview(self.getcontents()) + return newmemoryview(mv, mv.itemsize, '&' + mv.format, mv.shape) def _cast_addr(obj, _, tp): if not (isinstance(tp, _CDataMeta) and tp._is_pointer_like()): diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -2,9 +2,9 @@ import _rawffi from _ctypes.basics import _CData, _CDataMeta, keepalive_key,\ store_reference, ensure_objects, CArgObject -from _ctypes.array import Array +from _ctypes.array import Array, get_format_str from _ctypes.pointer import _Pointer -import inspect +import inspect, __pypy__ def names_and_fields(self, _fields_, superclass, anonymous_fields=None): @@ -176,6 +176,11 @@ class StructOrUnionMeta(_CDataMeta): def __new__(self, name, cls, typedict): res = type.__new__(self, name, cls, typedict) + if hasattr(res, '_swappedbytes_') and '_fields_' in typedict: + # Activate the stdlib ctypes._swapped_meta.__setattr__ to convert fields + tmp = res._fields_ + delattr(res, '_fields_') + setattr(res, '_fields_', tmp) if "_abstract_" in typedict: return res cls = cls or (object,) @@ -254,17 +259,7 @@ or cls is union.Union): raise TypeError("abstract class") if hasattr(cls, '_swappedbytes_'): - fields = [None] * len(cls._fields_) - for i in range(len(cls._fields_)): - if cls._fields_[i][1] == cls._fields_[i][1].__dict__.get('__ctype_be__', None): - swapped = cls._fields_[i][1].__dict__.get('__ctype_le__', cls._fields_[i][1]) - else: - swapped = cls._fields_[i][1].__dict__.get('__ctype_be__', cls._fields_[i][1]) - if len(cls._fields_[i]) < 3: - fields[i] = (cls._fields_[i][0], swapped) - else: - fields[i] = (cls._fields_[i][0], swapped, cls._fields_[i][2]) - names_and_fields(cls, fields, _CData, cls.__dict__.get('_anonymous_', None)) + names_and_fields(cls, cls._fields_, _CData, cls.__dict__.get('_anonymous_', None)) self = super(_CData, cls).__new__(cls) if hasattr(cls, '_ffistruct_'): self.__dict__['_buffer'] = self._ffistruct_(autofree=True) @@ -304,6 +299,10 @@ def _to_ffi_param(self): return self._buffer + def __buffer__(self, flags): + fmt = get_format_str(self) + itemsize = type(self)._sizeofinstances() + return __pypy__.newmemoryview(memoryview(self._buffer), itemsize, fmt) class StructureMeta(StructOrUnionMeta): _is_union = False diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst --- a/pypy/doc/__pypy__-module.rst +++ b/pypy/doc/__pypy__-module.rst @@ -13,7 +13,6 @@ ``if platform.python_implementation == 'PyPy'`` block or otherwise hidden from the CPython interpreter. - Generally available functionality --------------------------------- @@ -25,13 +24,14 @@ - ``attach_gdb()``: start a GDB at the interpreter-level (or a PDB before translation). - - ``identity_dict(object)``: A dictionary that considers keys by object identity. - Distinct objects will have separate entries even if they compare equal. - All objects can be used as keys, even non-hashable ones --- but avoid using - immutable objects like integers: two int objects 42 may or may not be - internally the same object. + - ``newmemoryview(buffer, itemsize, format, shape=None, strides=None)``: + create a `memoryview` instance with the data from ``buffer`` and the + specified itemsize, format, and optional shape and strides. - - ``set_debug`` + - ``bufferable``: a base class that provides a ``__buffer__(self, flags)`` + method for subclasses to override. This method should return a memoryview + instance of the class instance. It is called by the C-API's ``tp_as_buffer. + bf_getbuffer``. - ``builtinify(func)``: To implement at app-level modules that are, in CPython, implemented in C: this decorator protects a function from being ever bound diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -37,3 +37,9 @@ Use utf8 internally to represent unicode, with the goal of never using rpython-level unicode +.. branch: newmemoryview-app-level + +Since _ctypes is implemented in pure python over libffi, add interfaces and +methods to support the buffer interface from python. Specifically, add a +``__pypy__.newmemoryview`` function to create a memoryview and extend the use +of the PyPy-specific ``__buffer__`` class method. diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -62,11 +62,18 @@ class PyPyDateTime(MixedModule): appleveldefs = {} interpleveldefs = { - 'dateinterop': 'interp_pypydatetime.W_DateTime_Date', - 'timeinterop' : 'interp_pypydatetime.W_DateTime_Time', - 'deltainterop' : 'interp_pypydatetime.W_DateTime_Delta', + 'dateinterop' : 'interp_pypydatetime.W_DateTime_Date', + 'timeinterop' : 'interp_pypydatetime.W_DateTime_Time', + 'deltainterop' : 'interp_pypydatetime.W_DateTime_Delta', } +class PyPyBufferable(MixedModule): + appleveldefs = {} + interpleveldefs = { + 'bufferable': 'interp_buffer.W_Bufferable', + } + + class Module(MixedModule): """ PyPy specific "magic" functions. A lot of them are experimental and subject to change, many are internal. """ @@ -110,6 +117,7 @@ 'side_effects_ok' : 'interp_magic.side_effects_ok', 'stack_almost_full' : 'interp_magic.stack_almost_full', 'pyos_inputhook' : 'interp_magic.pyos_inputhook', + 'newmemoryview' : 'interp_buffer.newmemoryview', } if sys.platform == 'win32': interpleveldefs['get_console_cp'] = 'interp_magic.get_console_cp' @@ -121,6 +129,7 @@ "intop": IntOpModule, "os": OsModule, '_pypydatetime': PyPyDateTime, + 'bufferable': PyPyBufferable, } def setup_after_space_initialization(self): diff --git a/pypy/module/__pypy__/interp_buffer.py b/pypy/module/__pypy__/interp_buffer.py new file mode 100644 --- /dev/null +++ b/pypy/module/__pypy__/interp_buffer.py @@ -0,0 +1,100 @@ +# +# An app-level interface to tp_as_buffer->bf_getbuffer. +# + +from pypy.interpreter.error import oefmt +from pypy.interpreter.gateway import unwrap_spec, interp2app +from pypy.objspace.std.memoryobject import BufferViewND +from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.typedef import TypeDef, generic_new_descr + +class W_Bufferable(W_Root): + def __init__(self, space): + pass + + def descr_buffer(self, space, w_flags): + if type(self) is W_Bufferable: + raise oefmt(space.w_ValueError, "override __buffer__ in a subclass") + return space.call_method(self, '__buffer__', w_flags) + + def readbuf_w(self, space): + mv = space.call_method(self, '__buffer__', space.newint(0)) + return mv.buffer_w(space, 0).as_readbuf() + +W_Bufferable.typedef = TypeDef("Bufferable", None, None, 'read-write', + __doc__ = """a helper class for a app-level class (like _ctypes.Array) +that want to support tp_as_buffer.bf_getbuffer via a __buffer__ method""", + __new__ = generic_new_descr(W_Bufferable), + __buffer__ = interp2app(W_Bufferable.descr_buffer), +) + + at unwrap_spec(itemsize=int, format='text') +def newmemoryview(space, w_obj, itemsize, format, w_shape=None, w_strides=None): + ''' + newmemoryview(buf, itemsize, format, shape=None, strides=None) + ''' + if not space.isinstance_w(w_obj, space.w_memoryview): + raise oefmt(space.w_ValueError, "memoryview expected") + # minimal error checking + lgt = space.len_w(w_obj) + old_size = w_obj.getitemsize() + nbytes = lgt * old_size + if w_shape: + tot = 1 + shape = [] + for w_v in space.listview(w_shape): + v = space.int_w(w_v) + shape.append(v) + tot *= v + if tot * itemsize != nbytes: + raise oefmt(space.w_ValueError, + "shape/itemsize %s/%d does not match obj len/itemsize %d/%d", + str(shape), itemsize, lgt, old_size) + else: + if nbytes % itemsize != 0: + raise oefmt(space.w_ValueError, + "itemsize %d does not match obj len/itemsize %d/%d", + itemsize, lgt, old_size) + shape = [nbytes / itemsize,] + ndim = len(shape) + if w_strides: + strides = [] + for w_v in space.listview(w_strides): + v = space.int_w(w_v) + strides.append(v) + if not w_shape and len(strides) != 1: + raise oefmt(space.w_ValueError, + "strides must have one value if shape not provided") + if len(strides) != ndim: + raise oefmt(space.w_ValueError, + "shape %s does not match strides %s", + str(shape), str(strides)) + else: + # start from the right, c-order layout + strides = [itemsize] * ndim + for v in range(ndim - 2, -1, -1): + strides[v] = strides[v + 1] * shape[v + 1] + # check that the strides are not too big + for i in range(ndim): + if strides[i] * shape[i] > nbytes: + raise oefmt(space.w_ValueError, + "shape %s and strides %s exceed object size %d", + shape, strides, nbytes) + view = space.buffer_w(w_obj, 0) + return space.newmemoryview(FormatBufferViewND(view, itemsize, format, ndim, + shape, strides)) + +class FormatBufferViewND(BufferViewND): + _immutable_ = True + _attrs_ = ['readonly', 'parent', 'ndim', 'shape', 'strides', + 'format', 'itemsize'] + def __init__(self, parent, itemsize, format, ndim, shape, strides): + BufferViewND.__init__(self, parent, ndim, shape, strides) + self.format = format + self.itemsize = itemsize + + def getformat(self): + return self.format + + def getitemsize(self): + return self.itemsize diff --git a/pypy/module/__pypy__/test/test_newmemoryview.py b/pypy/module/__pypy__/test/test_newmemoryview.py new file mode 100644 --- /dev/null +++ b/pypy/module/__pypy__/test/test_newmemoryview.py @@ -0,0 +1,32 @@ + + +class AppTestMinimal: + spaceconfig = dict(usemodules=['__pypy__']) + + def test_newmemoryview(self): + from __pypy__ import newmemoryview + b = bytearray(12) + # The format can be anything, we only verify shape, strides, and itemsize + m = newmemoryview(memoryview(b), 2, 'T{ Author: Matti Picus Branch: py3.6 Changeset: r96233:bc8fb0e12474 Date: 2019-03-10 12:44 +0200 http://bitbucket.org/pypy/pypy/changeset/bc8fb0e12474/ Log: merge default into branch diff --git a/extra_tests/ctypes_tests/test_structures.py b/extra_tests/ctypes_tests/test_structures.py --- a/extra_tests/ctypes_tests/test_structures.py +++ b/extra_tests/ctypes_tests/test_structures.py @@ -119,12 +119,15 @@ ms.n = 0xff00 return repr(ba[:]) + nstruct = dostruct(Native) if sys.byteorder == 'little': - assert dostruct(Native) == dostruct(Little) - assert dostruct(Native) != dostruct(Big) + assert nstruct == dostruct(Little) + assert nstruct != dostruct(Big) + assert Big._fields_[0][1] is not i else: - assert dostruct(Native) == dostruct(Big) - assert dostruct(Native) != dostruct(Little) + assert nstruct == dostruct(Big) + assert nstruct != dostruct(Little) + assert Little._fields_[0][1] is not i def test_from_buffer_copy(): from array import array @@ -185,3 +188,20 @@ assert sizeof(s) == 3 * sizeof(c_int) assert s.a == 4 # 256 + 4 assert s.b == -123 + +def test_memoryview(): + class S(Structure): + _fields_ = [('a', c_int16), + ('b', c_int16), + ] + + S3 = S * 3 + c_array = (2 * S3)( + S3(S(a=0, b=1), S(a=2, b=3), S(a=4, b=5)), + S3(S(a=6, b=7), S(a=8, b=9), S(a=10, b=11)), + ) + + mv = memoryview(c_array) + assert mv.format == 'T{'} +swappedorder = {'little': '>', 'big': '<'} + +def get_format_str(typ): + if hasattr(typ, '_fields_'): + if hasattr(typ, '_swappedbytes_'): + bo = swappedorder[sys.byteorder] + else: + bo = byteorder[sys.byteorder] + flds = [] + for name, obj in typ._fields_: + # Trim off the leading '<' or '>' + ch = get_format_str(obj)[1:] + if (ch) == 'B': + flds.append(byteorder[sys.byteorder]) + else: + flds.append(bo) + flds.append(ch) + flds.append(':') + flds.append(name) + flds.append(':') + return 'T{' + ''.join(flds) + '}' + elif hasattr(typ, '_type_'): + ch = typ._type_ + return byteorder[sys.byteorder] + ch + else: + raise ValueError('cannot get format string for %r' % typ) diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py --- a/lib_pypy/_ctypes/basics.py +++ b/lib_pypy/_ctypes/basics.py @@ -2,8 +2,15 @@ from _rawffi import alt as _ffi import sys -try: from __pypy__ import builtinify -except ImportError: builtinify = lambda f: f +try: + from __pypy__ import builtinify +except ImportError: + builtinify = lambda f: f + +try: + from __pypy__.bufferable import bufferable +except ImportError: + bufferable = object keepalive_key = str # XXX fix this when provided with test @@ -64,7 +71,7 @@ 'resbuffer' is a _rawffi array of length 1 containing the value, and this returns a general Python object that corresponds. """ - res = object.__new__(self) + res = bufferable.__new__(self) res.__class__ = self res.__dict__['_buffer'] = resbuffer if base is not None: @@ -148,7 +155,7 @@ def __ne__(self, other): return self._obj != other -class _CData(object, metaclass=_CDataMeta): +class _CData(bufferable, metaclass=_CDataMeta): """ The most basic object for all ctypes types """ _objects = None diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py --- a/lib_pypy/_ctypes/pointer.py +++ b/lib_pypy/_ctypes/pointer.py @@ -7,8 +7,7 @@ from _ctypes.array import Array, array_get_slice_params, array_slice_getitem,\ array_slice_setitem -try: from __pypy__ import builtinify -except ImportError: builtinify = lambda f: f +from __pypy__ import builtinify, newmemoryview # This cache maps types to pointers to them. _pointer_type_cache = {} @@ -134,6 +133,9 @@ def _as_ffi_pointer_(self, ffitype): return as_ffi_pointer(self, ffitype) + def __buffer__(self, flags): + mv = memoryview(self.getcontents()) + return newmemoryview(mv, mv.itemsize, '&' + mv.format, mv.shape) def _cast_addr(obj, _, tp): if not (isinstance(tp, _CDataMeta) and tp._is_pointer_like()): diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -2,9 +2,9 @@ import _rawffi from _ctypes.basics import _CData, _CDataMeta, keepalive_key,\ store_reference, ensure_objects, CArgObject -from _ctypes.array import Array +from _ctypes.array import Array, get_format_str from _ctypes.pointer import _Pointer -import inspect +import inspect, __pypy__ def names_and_fields(self, _fields_, superclass, anonymous_fields=None): @@ -176,6 +176,11 @@ class StructOrUnionMeta(_CDataMeta): def __new__(self, name, cls, typedict): res = type.__new__(self, name, cls, typedict) + if hasattr(res, '_swappedbytes_') and '_fields_' in typedict: + # Activate the stdlib ctypes._swapped_meta.__setattr__ to convert fields + tmp = res._fields_ + delattr(res, '_fields_') + setattr(res, '_fields_', tmp) if "_abstract_" in typedict: return res cls = cls or (object,) @@ -253,17 +258,7 @@ or cls is union.Union): raise TypeError("abstract class") if hasattr(cls, '_swappedbytes_'): - fields = [None] * len(cls._fields_) - for i in range(len(cls._fields_)): - if cls._fields_[i][1] == cls._fields_[i][1].__dict__.get('__ctype_be__', None): - swapped = cls._fields_[i][1].__dict__.get('__ctype_le__', cls._fields_[i][1]) - else: - swapped = cls._fields_[i][1].__dict__.get('__ctype_be__', cls._fields_[i][1]) - if len(cls._fields_[i]) < 3: - fields[i] = (cls._fields_[i][0], swapped) - else: - fields[i] = (cls._fields_[i][0], swapped, cls._fields_[i][2]) - names_and_fields(cls, fields, _CData, cls.__dict__.get('_anonymous_', None)) + names_and_fields(cls, cls._fields_, _CData, cls.__dict__.get('_anonymous_', None)) self = super(_CData, cls).__new__(cls) if hasattr(cls, '_ffistruct_'): self.__dict__['_buffer'] = self._ffistruct_(autofree=True) @@ -303,6 +298,10 @@ def _to_ffi_param(self): return self._buffer + def __buffer__(self, flags): + fmt = get_format_str(self) + itemsize = type(self)._sizeofinstances() + return __pypy__.newmemoryview(memoryview(self._buffer), itemsize, fmt) class StructureMeta(StructOrUnionMeta): _is_union = False diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst --- a/pypy/doc/__pypy__-module.rst +++ b/pypy/doc/__pypy__-module.rst @@ -13,7 +13,6 @@ ``if platform.python_implementation == 'PyPy'`` block or otherwise hidden from the CPython interpreter. - Generally available functionality --------------------------------- @@ -25,13 +24,14 @@ - ``attach_gdb()``: start a GDB at the interpreter-level (or a PDB before translation). - - ``identity_dict(object)``: A dictionary that considers keys by object identity. - Distinct objects will have separate entries even if they compare equal. - All objects can be used as keys, even non-hashable ones --- but avoid using - immutable objects like integers: two int objects 42 may or may not be - internally the same object. + - ``newmemoryview(buffer, itemsize, format, shape=None, strides=None)``: + create a `memoryview` instance with the data from ``buffer`` and the + specified itemsize, format, and optional shape and strides. - - ``set_debug`` + - ``bufferable``: a base class that provides a ``__buffer__(self, flags)`` + method for subclasses to override. This method should return a memoryview + instance of the class instance. It is called by the C-API's ``tp_as_buffer. + bf_getbuffer``. - ``builtinify(func)``: To implement at app-level modules that are, in CPython, implemented in C: this decorator protects a function from being ever bound diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -32,3 +32,10 @@ .. branch: unicode-utf8 Use utf8 internally to represent unicode, with the goal of never using rpython-level unicode + +.. branch: newmemoryview-app-level + +Since _ctypes is implemented in pure python over libffi, add interfaces and +methods to support the buffer interface from python. Specifically, add a +``__pypy__.newmemoryview`` function to create a memoryview and extend the use +of the PyPy-specific ``__buffer__`` class method. \ No newline at end of file diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -62,11 +62,18 @@ class PyPyDateTime(MixedModule): appleveldefs = {} interpleveldefs = { - 'dateinterop': 'interp_pypydatetime.W_DateTime_Date', - 'timeinterop' : 'interp_pypydatetime.W_DateTime_Time', - 'deltainterop' : 'interp_pypydatetime.W_DateTime_Delta', + 'dateinterop' : 'interp_pypydatetime.W_DateTime_Date', + 'timeinterop' : 'interp_pypydatetime.W_DateTime_Time', + 'deltainterop' : 'interp_pypydatetime.W_DateTime_Delta', } +class PyPyBufferable(MixedModule): + appleveldefs = {} + interpleveldefs = { + 'bufferable': 'interp_buffer.W_Bufferable', + } + + class Module(MixedModule): """ PyPy specific "magic" functions. A lot of them are experimental and subject to change, many are internal. """ @@ -111,6 +118,7 @@ 'fsencode' : 'interp_magic.fsencode', 'fsdecode' : 'interp_magic.fsdecode', 'pyos_inputhook' : 'interp_magic.pyos_inputhook', + 'newmemoryview' : 'interp_buffer.newmemoryview', } submodules = { @@ -120,6 +128,7 @@ "intop": IntOpModule, "os": OsModule, '_pypydatetime': PyPyDateTime, + 'bufferable': PyPyBufferable, } def setup_after_space_initialization(self): diff --git a/pypy/module/__pypy__/interp_buffer.py b/pypy/module/__pypy__/interp_buffer.py new file mode 100644 --- /dev/null +++ b/pypy/module/__pypy__/interp_buffer.py @@ -0,0 +1,100 @@ +# +# An app-level interface to tp_as_buffer->bf_getbuffer. +# + +from pypy.interpreter.error import oefmt +from pypy.interpreter.gateway import unwrap_spec, interp2app +from pypy.objspace.std.memoryobject import BufferViewND +from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.typedef import TypeDef, generic_new_descr + +class W_Bufferable(W_Root): + def __init__(self, space): + pass + + def descr_buffer(self, space, w_flags): + if type(self) is W_Bufferable: + raise oefmt(space.w_ValueError, "override __buffer__ in a subclass") + return space.call_method(self, '__buffer__', w_flags) + + def readbuf_w(self, space): + mv = space.call_method(self, '__buffer__', space.newint(0)) + return mv.buffer_w(space, 0).as_readbuf() + +W_Bufferable.typedef = TypeDef("Bufferable", None, None, 'read-write', + __doc__ = """a helper class for a app-level class (like _ctypes.Array) +that want to support tp_as_buffer.bf_getbuffer via a __buffer__ method""", + __new__ = generic_new_descr(W_Bufferable), + __buffer__ = interp2app(W_Bufferable.descr_buffer), +) + + at unwrap_spec(itemsize=int, format='text') +def newmemoryview(space, w_obj, itemsize, format, w_shape=None, w_strides=None): + ''' + newmemoryview(buf, itemsize, format, shape=None, strides=None) + ''' + if not space.isinstance_w(w_obj, space.w_memoryview): + raise oefmt(space.w_ValueError, "memoryview expected") + # minimal error checking + lgt = space.len_w(w_obj) + old_size = w_obj.getitemsize() + nbytes = lgt * old_size + if w_shape: + tot = 1 + shape = [] + for w_v in space.listview(w_shape): + v = space.int_w(w_v) + shape.append(v) + tot *= v + if tot * itemsize != nbytes: + raise oefmt(space.w_ValueError, + "shape/itemsize %s/%d does not match obj len/itemsize %d/%d", + str(shape), itemsize, lgt, old_size) + else: + if nbytes % itemsize != 0: + raise oefmt(space.w_ValueError, + "itemsize %d does not match obj len/itemsize %d/%d", + itemsize, lgt, old_size) + shape = [nbytes / itemsize,] + ndim = len(shape) + if w_strides: + strides = [] + for w_v in space.listview(w_strides): + v = space.int_w(w_v) + strides.append(v) + if not w_shape and len(strides) != 1: + raise oefmt(space.w_ValueError, + "strides must have one value if shape not provided") + if len(strides) != ndim: + raise oefmt(space.w_ValueError, + "shape %s does not match strides %s", + str(shape), str(strides)) + else: + # start from the right, c-order layout + strides = [itemsize] * ndim + for v in range(ndim - 2, -1, -1): + strides[v] = strides[v + 1] * shape[v + 1] + # check that the strides are not too big + for i in range(ndim): + if strides[i] * shape[i] > nbytes: + raise oefmt(space.w_ValueError, + "shape %s and strides %s exceed object size %d", + shape, strides, nbytes) + view = space.buffer_w(w_obj, 0) + return space.newmemoryview(FormatBufferViewND(view, itemsize, format, ndim, + shape, strides)) + +class FormatBufferViewND(BufferViewND): + _immutable_ = True + _attrs_ = ['readonly', 'parent', 'ndim', 'shape', 'strides', + 'format', 'itemsize'] + def __init__(self, parent, itemsize, format, ndim, shape, strides): + BufferViewND.__init__(self, parent, ndim, shape, strides) + self.format = format + self.itemsize = itemsize + + def getformat(self): + return self.format + + def getitemsize(self): + return self.itemsize diff --git a/pypy/module/__pypy__/test/test_newmemoryview.py b/pypy/module/__pypy__/test/test_newmemoryview.py new file mode 100644 --- /dev/null +++ b/pypy/module/__pypy__/test/test_newmemoryview.py @@ -0,0 +1,32 @@ + + +class AppTestMinimal: + spaceconfig = dict(usemodules=['__pypy__']) + + def test_newmemoryview(self): + from __pypy__ import newmemoryview + b = bytearray(12) + # The format can be anything, we only verify shape, strides, and itemsize + m = newmemoryview(memoryview(b), 2, 'T{ Author: Matti Picus Branch: issue2968 Changeset: r96234:cf058300ea04 Date: 2019-03-10 21:54 +0200 http://bitbucket.org/pypy/pypy/changeset/cf058300ea04/ Log: move PyTuple_Type.tp_new to C diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -1189,7 +1189,9 @@ state.C.get_pyos_inputhook = rffi.llexternal( '_PyPy_get_PyOS_InputHook', [], FUNCPTR, compilation_info=eci, _nowrapper=True) - + state.C.tuple_new = rffi.llexternal( + 'tuple_new', [PyTypeObjectPtr, PyObject, PyObject], PyObject, + compilation_info=eci, _nowrapper=True) def init_function(func): INIT_FUNCTIONS.append(func) diff --git a/pypy/module/cpyext/include/tupleobject.h b/pypy/module/cpyext/include/tupleobject.h --- a/pypy/module/cpyext/include/tupleobject.h +++ b/pypy/module/cpyext/include/tupleobject.h @@ -18,6 +18,7 @@ PyAPI_FUNC(PyObject *) PyTuple_New(Py_ssize_t size); PyAPI_FUNC(void) _PyPy_tuple_dealloc(PyObject *); +PyAPI_FUNC(PyObject *) tuple_new(PyTypeObject *type, PyObject *args, PyObject *kwds); /* defined in varargswrapper.c */ PyAPI_FUNC(PyObject *) PyTuple_Pack(Py_ssize_t, ...); diff --git a/pypy/module/cpyext/src/tupleobject.c b/pypy/module/cpyext/src/tupleobject.c --- a/pypy/module/cpyext/src/tupleobject.c +++ b/pypy/module/cpyext/src/tupleobject.c @@ -89,3 +89,48 @@ done: Py_TRASHCAN_SAFE_END(op) } + +static PyObject * +tuple_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds); + +PyObject * +tuple_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyObject *arg = NULL; + static char *kwlist[] = {"sequence", 0}; + + if (type != &PyTuple_Type) + return tuple_subtype_new(type, args, kwds); + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O:tuple", kwlist, &arg)) + return NULL; + + if (arg == NULL) + return PyTuple_New(0); + else + return PySequence_Tuple(arg); +} + +static PyObject * +tuple_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyObject *tmp, *newobj, *item; + Py_ssize_t i, n; + + assert(PyType_IsSubtype(type, &PyTuple_Type)); + tmp = tuple_new(&PyTuple_Type, args, kwds); + if (tmp == NULL) + return NULL; + assert(PyTuple_Check(tmp)); + newobj = type->tp_alloc(type, n = PyTuple_GET_SIZE(tmp)); + if (newobj == NULL) + return NULL; + for (i = 0; i < n; i++) { + item = PyTuple_GET_ITEM(tmp, i); + Py_INCREF(item); + PyTuple_SET_ITEM(newobj, i, item); + } + Py_DECREF(tmp); + return newobj; +} + + diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -686,6 +686,10 @@ update_all_slots(space, w_type, pto) else: update_all_slots_builtin(space, w_type, pto) + + if space.is_w(w_type, space.w_tuple): + pto.c_tp_new = state.C.tuple_new + if not pto.c_tp_new: base_object_pyo = make_ref(space, space.w_object) base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo) From pypy.commits at gmail.com Sun Mar 10 16:45:22 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:45:22 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head 284828e44f37 on branch 35windowsfixes Message-ID: <5c857762.1c69fb81.182d7.fc04@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96235:2150cd15e853 Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/2150cd15e853/ Log: Merge closed head 284828e44f37 on branch 35windowsfixes From pypy.commits at gmail.com Sun Mar 10 16:45:24 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:45:24 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head ce962d90778a on branch sys-getsizeof Message-ID: <5c857764.1c69fb81.5d06a.8381@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96236:894158740b41 Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/894158740b41/ Log: Merge closed head ce962d90778a on branch sys-getsizeof From pypy.commits at gmail.com Sun Mar 10 16:45:25 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:45:25 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head 147bb1ff2239 on branch fix-longevity Message-ID: <5c857765.1c69fb81.b213d.2c26@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96237:d8fcf1ee1457 Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/d8fcf1ee1457/ Log: Merge closed head 147bb1ff2239 on branch fix-longevity From pypy.commits at gmail.com Sun Mar 10 16:45:27 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:45:27 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head a343830b763a on branch regalloc Message-ID: <5c857767.1c69fb81.cd229.712f@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96238:bb1dba46290a Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/bb1dba46290a/ Log: Merge closed head a343830b763a on branch regalloc From pypy.commits at gmail.com Sun Mar 10 16:45:29 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:45:29 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head ce5a8c7604a6 on branch rmod-radd-slots Message-ID: <5c857769.1c69fb81.44bbf.8815@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96239:51af293c28b5 Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/51af293c28b5/ Log: Merge closed head ce5a8c7604a6 on branch rmod-radd-slots From pypy.commits at gmail.com Sun Mar 10 16:45:30 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:45:30 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head cf24682f0f6d on branch ndarray-promote Message-ID: <5c85776a.1c69fb81.4053f.1375@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96240:dca1850261e0 Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/dca1850261e0/ Log: Merge closed head cf24682f0f6d on branch ndarray-promote From pypy.commits at gmail.com Sun Mar 10 16:45:32 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:45:32 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head 7cf7426ddc77 on branch pypy3-release-2.6.x Message-ID: <5c85776c.1c69fb81.77c88.3320@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96241:7c26677025b0 Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/7c26677025b0/ Log: Merge closed head 7cf7426ddc77 on branch pypy3-release-2.6.x From pypy.commits at gmail.com Sun Mar 10 16:45:33 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:45:33 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head 25560ec3b2f5 on branch pypy3-release-2.3.x Message-ID: <5c85776d.1c69fb81.81388.8fbd@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96242:43fd44792fc3 Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/43fd44792fc3/ Log: Merge closed head 25560ec3b2f5 on branch pypy3-release-2.3.x From pypy.commits at gmail.com Sun Mar 10 16:45:35 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:45:35 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head 072df3de15c5 on branch pypy3-release-2.4.x Message-ID: <5c85776f.1c69fb81.f9410.a09d@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96243:ee67fab05f0c Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/ee67fab05f0c/ Log: Merge closed head 072df3de15c5 on branch pypy3-release-2.4.x From pypy.commits at gmail.com Sun Mar 10 16:45:36 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:45:36 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head 6dfb3af9716d on branch release-pypy3.3-v5 Message-ID: <5c857770.1c69fb81.155bf.f762@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96244:2552f5eb61e9 Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/2552f5eb61e9/ Log: Merge closed head 6dfb3af9716d on branch release-pypy3.3-v5 From pypy.commits at gmail.com Sun Mar 10 16:45:38 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:45:38 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head 1ce3b640e7d7 on branch release-5.x Message-ID: <5c857772.1c69fb81.767f3.1fda@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96245:fc8d274f73f2 Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/fc8d274f73f2/ Log: Merge closed head 1ce3b640e7d7 on branch release-5.x From pypy.commits at gmail.com Sun Mar 10 16:45:40 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:45:40 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head a32aff107924 on branch numpy_broadcast_nd Message-ID: <5c857774.1c69fb81.db0ce.f595@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96246:61815eabc76d Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/61815eabc76d/ Log: Merge closed head a32aff107924 on branch numpy_broadcast_nd From pypy.commits at gmail.com Sun Mar 10 16:45:41 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:45:41 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head fba889ae9aaa on branch cpyext-inheritance Message-ID: <5c857775.1c69fb81.7181e.ec7c@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96247:8f0676ce5d7a Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/8f0676ce5d7a/ Log: Merge closed head fba889ae9aaa on branch cpyext-inheritance From pypy.commits at gmail.com Sun Mar 10 16:45:43 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:45:43 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head 91b5766bb8c6 on branch cpyext-debug-type_dealloc Message-ID: <5c857777.1c69fb81.66299.cab5@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96248:55551010c78e Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/55551010c78e/ Log: Merge closed head 91b5766bb8c6 on branch cpyext-debug-type_dealloc From pypy.commits at gmail.com Sun Mar 10 16:45:44 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:45:44 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head 6e767f90b99a on branch override-tp_as-methods Message-ID: <5c857778.1c69fb81.c1c0c.cc0d@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96249:ee09d4329328 Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/ee09d4329328/ Log: Merge closed head 6e767f90b99a on branch override-tp_as-methods From pypy.commits at gmail.com Sun Mar 10 16:45:46 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:45:46 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head 6fcafa0bb5ea on branch matplotlib Message-ID: <5c85777a.1c69fb81.7181e.ec8d@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96250:88c6ba937e8b Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/88c6ba937e8b/ Log: Merge closed head 6fcafa0bb5ea on branch matplotlib From pypy.commits at gmail.com Sun Mar 10 16:45:47 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:45:47 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head e4e1582c4390 on branch win32-vmprof Message-ID: <5c85777b.1c69fb81.cc2f9.6d59@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96251:f9d6dddd78f1 Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/f9d6dddd78f1/ Log: Merge closed head e4e1582c4390 on branch win32-vmprof From pypy.commits at gmail.com Sun Mar 10 16:45:49 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:45:49 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head fa16a566a0a7 on branch non-linux-vmprof-stacklet-switch-2 Message-ID: <5c85777d.1c69fb81.3fb9b.96c2@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96252:0490ef76c38f Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/0490ef76c38f/ Log: Merge closed head fa16a566a0a7 on branch non-linux-vmprof-stacklet- switch-2 From pypy.commits at gmail.com Sun Mar 10 16:45:51 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:45:51 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head 0cdae6b1a07e on branch Opcode-class Message-ID: <5c85777f.1c69fb81.53f14.23ce@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96253:38768fe956bd Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/38768fe956bd/ Log: Merge closed head 0cdae6b1a07e on branch Opcode-class From pypy.commits at gmail.com Sun Mar 10 16:45:52 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:45:52 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head 879181847bd8 on branch taskengine-sorted-optionals Message-ID: <5c857780.1c69fb81.5d06a.83a2@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96254:8c91b895e8d3 Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/8c91b895e8d3/ Log: Merge closed head 879181847bd8 on branch taskengine-sorted-optionals From pypy.commits at gmail.com Sun Mar 10 16:45:54 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:45:54 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head 468c9599a1f6 on branch inline-taskengine Message-ID: <5c857782.1c69fb81.58c81.02fc@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96255:c215cb57666d Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/c215cb57666d/ Log: Merge closed head 468c9599a1f6 on branch inline-taskengine From pypy.commits at gmail.com Sun Mar 10 16:45:55 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:45:55 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head 7dc47b5a8a95 on branch numpypy-ctypes Message-ID: <5c857783.1c69fb81.b213d.2c6a@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96256:92b575c0af37 Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/92b575c0af37/ Log: Merge closed head 7dc47b5a8a95 on branch numpypy-ctypes From pypy.commits at gmail.com Sun Mar 10 16:45:57 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:45:57 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head 32693f76ec8f on branch numpy-record-type-pure-python Message-ID: <5c857785.1c69fb81.f9410.a0c8@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96257:227b513270c3 Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/227b513270c3/ Log: Merge closed head 32693f76ec8f on branch numpy-record-type-pure- python From pypy.commits at gmail.com Sun Mar 10 16:45:58 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:45:58 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head 877a16704f95 on branch struct-double Message-ID: <5c857786.1c69fb81.23915.2ca2@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96258:de74efc8d8f8 Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/de74efc8d8f8/ Log: Merge closed head 877a16704f95 on branch struct-double From pypy.commits at gmail.com Sun Mar 10 16:46:00 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:46:00 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head 5b69b3275f06 on branch dynamic-specialized-tuple Message-ID: <5c857788.1c69fb81.c1450.da58@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96259:55cb4adad1e2 Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/55cb4adad1e2/ Log: Merge closed head 5b69b3275f06 on branch dynamic-specialized-tuple From pypy.commits at gmail.com Sun Mar 10 16:46:01 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:46:01 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head b1f2ea41a0c5 on branch jit-sys-exc-info Message-ID: <5c857789.1c69fb81.80589.022d@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96260:5d7578d2dce3 Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/5d7578d2dce3/ Log: Merge closed head b1f2ea41a0c5 on branch jit-sys-exc-info From pypy.commits at gmail.com Sun Mar 10 16:46:03 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:46:03 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head 2bb6bffd210f on branch rpath-enforceargs Message-ID: <5c85778b.1c69fb81.b65ed.1645@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96261:1a57a458dd03 Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/1a57a458dd03/ Log: Merge closed head 2bb6bffd210f on branch rpath-enforceargs From pypy.commits at gmail.com Sun Mar 10 16:46:04 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:46:04 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head cf5840d4cb0f on branch more_strategies Message-ID: <5c85778c.1c69fb81.b3e9f.5df0@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96262:e188b090ff3d Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/e188b090ff3d/ Log: Merge closed head cf5840d4cb0f on branch more_strategies From pypy.commits at gmail.com Sun Mar 10 16:46:06 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:46:06 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head d417efd34983 on branch pread/pwrite Message-ID: <5c85778e.1c69fb81.eca2f.1d94@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96263:1b34a721a3f1 Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/1b34a721a3f1/ Log: Merge closed head d417efd34983 on branch pread/pwrite From pypy.commits at gmail.com Sun Mar 10 16:46:08 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:46:08 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: Merge closed head eb6e70a9658e on branch py3.5 Message-ID: <5c857790.1c69fb81.d6665.eac9@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96264:e1aa8ba118c3 Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/e1aa8ba118c3/ Log: Merge closed head eb6e70a9658e on branch py3.5 From pypy.commits at gmail.com Sun Mar 10 16:46:09 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:46:09 -0700 (PDT) Subject: [pypy-commit] pypy closed-branches: re-close this branch Message-ID: <5c857791.1c69fb81.c334e.1e51@mx.google.com> Author: Matti Picus Branch: closed-branches Changeset: r96265:e00407718a6e Date: 2019-03-10 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/e00407718a6e/ Log: re-close this branch From pypy.commits at gmail.com Sun Mar 10 16:46:11 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:46:11 -0700 (PDT) Subject: [pypy-commit] pypy newmemoryview-app-level: close merged branch Message-ID: <5c857793.1c69fb81.52f17.bad4@mx.google.com> Author: Matti Picus Branch: newmemoryview-app-level Changeset: r96266:9fbc2b4e5761 Date: 2019-03-10 22:43 +0200 http://bitbucket.org/pypy/pypy/changeset/9fbc2b4e5761/ Log: close merged branch From pypy.commits at gmail.com Sun Mar 10 16:46:12 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 13:46:12 -0700 (PDT) Subject: [pypy-commit] pypy default: merge closed branch Message-ID: <5c857794.1c69fb81.12a8d.66c9@mx.google.com> Author: Matti Picus Branch: Changeset: r96267:c531ef4097ad Date: 2019-03-10 22:44 +0200 http://bitbucket.org/pypy/pypy/changeset/c531ef4097ad/ Log: merge closed branch From pypy.commits at gmail.com Mon Mar 11 02:57:09 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 10 Mar 2019 23:57:09 -0700 (PDT) Subject: [pypy-commit] pypy default: typo Message-ID: <5c8606c5.1c69fb81.190c7.3c64@mx.google.com> Author: Matti Picus Branch: Changeset: r96268:9a56aa4669ee Date: 2019-03-11 08:56 +0200 http://bitbucket.org/pypy/pypy/changeset/9a56aa4669ee/ Log: typo diff --git a/pypy/module/_multiprocessing/test/test_win32.py b/pypy/module/_multiprocessing/test/test_win32.py --- a/pypy/module/_multiprocessing/test/test_win32.py +++ b/pypy/module/_multiprocessing/test/test_win32.py @@ -2,7 +2,7 @@ import sys class AppTestWin32: - spaceconfig = dict(usemodules=('_multiprocessing', _cffi_backend', + spaceconfig = dict(usemodules=('_multiprocessing', '_cffi_backend', 'signal', '_rawffi', 'binascii')) def setup_class(cls): From pypy.commits at gmail.com Mon Mar 11 05:08:11 2019 From: pypy.commits at gmail.com (arigo) Date: Mon, 11 Mar 2019 02:08:11 -0700 (PDT) Subject: [pypy-commit] cffi default: Issue #405 Message-ID: <5c86257b.1c69fb81.39a66.7943@mx.google.com> Author: Armin Rigo Branch: Changeset: r3240:0ad3630d7fb3 Date: 2019-03-11 10:09 +0100 http://bitbucket.org/cffi/cffi/changeset/0ad3630d7fb3/ Log: Issue #405 Fix for nested struct types that end in a var-sized array diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -174,7 +174,7 @@ #define CT_IS_BOOL 0x00080000 #define CT_IS_FILE 0x00100000 #define CT_IS_VOID_PTR 0x00200000 -#define CT_WITH_VAR_ARRAY 0x00400000 +#define CT_WITH_VAR_ARRAY 0x00400000 /* with open-ended array, anywhere */ /* unused 0x00800000 */ #define CT_LAZY_FIELD_LIST 0x01000000 #define CT_WITH_PACKED_CHANGE 0x02000000 @@ -1331,6 +1331,29 @@ } static int +add_varsize_length(Py_ssize_t offset, Py_ssize_t itemsize, + Py_ssize_t varsizelength, Py_ssize_t *optvarsize) +{ + /* update '*optvarsize' to account for an array of 'varsizelength' + elements, each of size 'itemsize', that starts at 'offset'. */ + Py_ssize_t size = ADD_WRAPAROUND(offset, + MUL_WRAPAROUND(itemsize, varsizelength)); + if (size < 0 || + ((size - offset) / itemsize) != varsizelength) { + PyErr_SetString(PyExc_OverflowError, + "array size would overflow a Py_ssize_t"); + return -1; + } + if (size > *optvarsize) + *optvarsize = size; + return 0; +} + +static int +convert_struct_from_object(char *data, CTypeDescrObject *ct, PyObject *init, + Py_ssize_t *optvarsize); /* forward */ + +static int convert_vfield_from_object(char *data, CFieldObject *cf, PyObject *value, Py_ssize_t *optvarsize) { @@ -1343,20 +1366,11 @@ if (optvarsize != NULL) { /* in this mode, the only purpose of this function is to compute the real size of the structure from a var-sized C99 array */ - Py_ssize_t size, itemsize; assert(data == NULL); - itemsize = cf->cf_type->ct_itemdescr->ct_size; - size = ADD_WRAPAROUND(cf->cf_offset, - MUL_WRAPAROUND(itemsize, varsizelength)); - if (size < 0 || - ((size - cf->cf_offset) / itemsize) != varsizelength) { - PyErr_SetString(PyExc_OverflowError, - "array size would overflow a Py_ssize_t"); - return -1; - } - if (size > *optvarsize) - *optvarsize = size; - return 0; + return add_varsize_length(cf->cf_offset, + cf->cf_type->ct_itemdescr->ct_size, + varsizelength, + optvarsize); } /* if 'value' was only an integer, get_new_array_length() returns it and convert 'value' to be None. Detect if this was the case, @@ -1365,8 +1379,16 @@ if (value == Py_None) return 0; } - if (optvarsize == NULL) + if (optvarsize == NULL) { return convert_field_from_object(data, cf, value); + } + else if ((cf->cf_type->ct_flags & CT_WITH_VAR_ARRAY) != 0 && + !CData_Check(value)) { + Py_ssize_t subsize = cf->cf_type->ct_size; + if (convert_struct_from_object(NULL, cf->cf_type, value, &subsize) < 0) + return -1; + return add_varsize_length(cf->cf_offset, 1, subsize, optvarsize); + } else return 0; } @@ -4951,6 +4973,15 @@ goto error; } } + else if (ftype->ct_flags & CT_WITH_VAR_ARRAY) { + /* GCC (or maybe C99) accepts var-sized struct fields that are not + the last field of a larger struct. That's why there is no + check here for "last field": we propagate the flag + CT_WITH_VAR_ARRAY to any struct that contains either an open- + ended array or another struct that recursively contains an + open-ended array. */ + ct->ct_flags |= CT_WITH_VAR_ARRAY; + } if (is_union) boffset = 0; /* reset each field at offset 0 */ diff --git a/testing/cffi0/test_ffi_backend.py b/testing/cffi0/test_ffi_backend.py --- a/testing/cffi0/test_ffi_backend.py +++ b/testing/cffi0/test_ffi_backend.py @@ -129,6 +129,36 @@ alloc5 = ffi.new_allocator(myalloc5) py.test.raises(MemoryError, alloc5, "int[5]") + def test_new_struct_containing_struct_containing_array_varsize(self): + ffi = FFI(backend=self.Backend()) + ffi.cdef(""" + struct foo_s { int len[100]; short data[]; }; + struct bar_s { int abc[100]; struct foo_s tail; }; + """) + # loop to try to detect heap overwrites, if the size allocated + # is too small + for i in range(1, 501, 100): + p = ffi.new("struct bar_s *", [[10], [[20], [3,4,5,6,7,8,9] * i]]) + assert p.abc[0] == 10 + assert p.tail.len[0] == 20 + assert p.tail.data[0] == 3 + assert p.tail.data[6] == 9 + assert p.tail.data[7 * i - 1] == 9 + + def test_bogus_struct_containing_struct_containing_array_varsize(self): + ffi = FFI(backend=self.Backend()) + ffi.cdef(""" + struct foo_s { signed char len; signed char data[]; }; + struct bar_s { struct foo_s foo; int bcd; }; + """) + p = ffi.new("struct bar_s *", [[123, [45, 56, 67, 78]], 9999999]) + assert p.foo.len == 123 + assert p.foo.data[0] == 45 + assert p.foo.data[1] == 56 + assert p.foo.data[2] == 67 + assert p.bcd == 9999999 + assert p.foo.data[3] != 78 # has been overwritten with 9999999 + class TestBitfield: def check(self, source, expected_ofs_y, expected_align, expected_size): From pypy.commits at gmail.com Mon Mar 11 05:18:11 2019 From: pypy.commits at gmail.com (arigo) Date: Mon, 11 Mar 2019 02:18:11 -0700 (PDT) Subject: [pypy-commit] cffi default: Add an extra test directly here, for pypy Message-ID: <5c8627d3.1c69fb81.cab74.968b@mx.google.com> Author: Armin Rigo Branch: Changeset: r3241:042594ebfddc Date: 2019-03-11 10:19 +0100 http://bitbucket.org/cffi/cffi/changeset/042594ebfddc/ Log: Add an extra test directly here, for pypy diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -3452,6 +3452,15 @@ assert p.a[1] == 20 assert p.a[2] == 30 assert p.a[3] == 0 + # + # struct of struct of varsized array + BStruct2 = new_struct_type("bar") + complete_struct_or_union(BStruct2, [('head', BInt), + ('tail', BStruct)]) + for i in range(2): # try to detect heap overwrites + p = newp(new_pointer_type(BStruct2), [100, [200, list(range(50))]]) + assert p.tail.y[49] == 49 + def test_struct_array_no_length_explicit_position(): BInt = new_primitive_type("int") From pypy.commits at gmail.com Mon Mar 11 05:39:44 2019 From: pypy.commits at gmail.com (arigo) Date: Mon, 11 Mar 2019 02:39:44 -0700 (PDT) Subject: [pypy-commit] cffi default: Fix (corner case, hard to test) Message-ID: <5c862ce0.1c69fb81.e4fcc.5749@mx.google.com> Author: Armin Rigo Branch: Changeset: r3242:ff25b4e68195 Date: 2019-03-11 10:27 +0100 http://bitbucket.org/cffi/cffi/changeset/ff25b4e68195/ Log: Fix (corner case, hard to test) diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -4973,14 +4973,18 @@ goto error; } } - else if (ftype->ct_flags & CT_WITH_VAR_ARRAY) { + else if (ftype->ct_flags & (CT_STRUCT|CT_UNION)) { + if (force_lazy_struct(ftype) < 0) /* for CT_WITH_VAR_ARRAY */ + return NULL; + /* GCC (or maybe C99) accepts var-sized struct fields that are not the last field of a larger struct. That's why there is no check here for "last field": we propagate the flag CT_WITH_VAR_ARRAY to any struct that contains either an open- ended array or another struct that recursively contains an open-ended array. */ - ct->ct_flags |= CT_WITH_VAR_ARRAY; + if (ftype->ct_flags & CT_WITH_VAR_ARRAY) + ct->ct_flags |= CT_WITH_VAR_ARRAY; } if (is_union) From pypy.commits at gmail.com Mon Mar 11 05:48:14 2019 From: pypy.commits at gmail.com (arigo) Date: Mon, 11 Mar 2019 02:48:14 -0700 (PDT) Subject: [pypy-commit] pypy default: update to cffi/ff25b4e68195 Message-ID: <5c862ede.1c69fb81.3b840.7b2a@mx.google.com> Author: Armin Rigo Branch: Changeset: r96269:fd0c6116edcd Date: 2019-03-11 10:46 +0100 http://bitbucket.org/pypy/pypy/changeset/fd0c6116edcd/ Log: update to cffi/ff25b4e68195 diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py --- a/pypy/module/_cffi_backend/ctypestruct.py +++ b/pypy/module/_cffi_backend/ctypestruct.py @@ -238,26 +238,32 @@ else: self.ctype.convert_from_object(cdata, w_ob) + def add_varsize_length(self, space, itemsize, varsizelength, optvarsize): + # returns an updated 'optvarsize' to account for an array of + # 'varsizelength' elements, each of size 'itemsize', that starts + # at 'self.offset'. + try: + varsize = ovfcheck(itemsize * varsizelength) + size = ovfcheck(self.offset + varsize) + except OverflowError: + raise oefmt(space.w_OverflowError, + "array size would overflow a ssize_t") + assert size >= 0 + return max(size, optvarsize) + def write_v(self, cdata, w_ob, optvarsize): # a special case for var-sized C99 arrays from pypy.module._cffi_backend import ctypearray ct = self.ctype + space = ct.space if isinstance(ct, ctypearray.W_CTypeArray) and ct.length < 0: - space = ct.space w_ob, varsizelength = ct.get_new_array_length(w_ob) if optvarsize != -1: # in this mode, the only purpose of this function is to compute # the real size of the structure from a var-sized C99 array assert cdata == lltype.nullptr(rffi.CCHARP.TO) - itemsize = ct.ctitem.size - try: - varsize = ovfcheck(itemsize * varsizelength) - size = ovfcheck(self.offset + varsize) - except OverflowError: - raise oefmt(space.w_OverflowError, - "array size would overflow a ssize_t") - assert size >= 0 - return max(size, optvarsize) + return self.add_varsize_length(space, ct.ctitem.size, + varsizelength, optvarsize) # if 'value' was only an integer, get_new_array_length() returns # w_ob = space.w_None. Detect if this was the case, # and if so, stop here, leaving the content uninitialized @@ -267,6 +273,12 @@ # if optvarsize == -1: self.write(cdata, w_ob) + elif (isinstance(ct, W_CTypeStructOrUnion) and ct._with_var_array and + not isinstance(w_ob, cdataobj.W_CData)): + subsize = ct.size + subsize = ct.convert_struct_from_object( + lltype.nullptr(rffi.CCHARP.TO), w_ob, subsize) + optvarsize = self.add_varsize_length(space, 1, subsize, optvarsize) return optvarsize def convert_bitfield_to_object(self, cdata): diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py --- a/pypy/module/_cffi_backend/newtype.py +++ b/pypy/module/_cffi_backend/newtype.py @@ -368,6 +368,16 @@ raise oefmt(space.w_TypeError, "field '%s.%s' has ctype '%s' of unknown size", w_ctype.name, fname, ftype.name) + elif isinstance(ftype, ctypestruct.W_CTypeStructOrUnion): + ftype.force_lazy_struct() + # GCC (or maybe C99) accepts var-sized struct fields that are not + # the last field of a larger struct. That's why there is no + # check here for "last field": we propagate the flag + # '_with_var_array' to any struct that contains either an open- + # ended array or another struct that recursively contains an + # open-ended array. + if ftype._with_var_array: + with_var_array = True # if is_union: boffset = 0 # reset each field at offset 0 @@ -419,7 +429,6 @@ # a nested anonymous struct or union # note: it seems we only get here with ffi.verify() srcfield2names = {} - ftype.force_lazy_struct() for name, srcfld in ftype._fields_dict.items(): srcfield2names[srcfld] = name for srcfld in ftype._fields_list: diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -3441,6 +3441,15 @@ assert p.a[1] == 20 assert p.a[2] == 30 assert p.a[3] == 0 + # + # struct of struct of varsized array + BStruct2 = new_struct_type("bar") + complete_struct_or_union(BStruct2, [('head', BInt), + ('tail', BStruct)]) + for i in range(2): # try to detect heap overwrites + p = newp(new_pointer_type(BStruct2), [100, [200, list(range(50))]]) + assert p.tail.y[49] == 49 + def test_struct_array_no_length_explicit_position(): BInt = new_primitive_type("int") From pypy.commits at gmail.com Mon Mar 11 06:40:26 2019 From: pypy.commits at gmail.com (mattip) Date: Mon, 11 Mar 2019 03:40:26 -0700 (PDT) Subject: [pypy-commit] pypy default: fix test Message-ID: <5c863b1a.1c69fb81.c87ea.ff1e@mx.google.com> Author: Matti Picus Branch: Changeset: r96270:c035c7de5e39 Date: 2019-03-11 09:12 +0200 http://bitbucket.org/pypy/pypy/changeset/c035c7de5e39/ Log: fix test diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py --- a/lib_pypy/_ctypes/array.py +++ b/lib_pypy/_ctypes/array.py @@ -257,7 +257,7 @@ try: itemsize = struct.calcsize(fmt[1:]) except: - itemsize = len(buffer(obj[0])) + itemsize = sizeof(obj[0]) return __pypy__.newmemoryview(memoryview(self._buffer), itemsize, fmt, shape) ARRAY_CACHE = {} diff --git a/pypy/module/__pypy__/test/test_newmemoryview.py b/pypy/module/__pypy__/test/test_newmemoryview.py --- a/pypy/module/__pypy__/test/test_newmemoryview.py +++ b/pypy/module/__pypy__/test/test_newmemoryview.py @@ -19,7 +19,7 @@ from __pypy__ import bufferable, newmemoryview class B(bufferable.bufferable): def __init__(self): - self.data = bytearray('abc') + self.data = bytearray(b'abc') def __buffer__(self, flags): return newmemoryview(memoryview(self.data), 1, 'B') From pypy.commits at gmail.com Mon Mar 11 08:14:39 2019 From: pypy.commits at gmail.com (mattip) Date: Mon, 11 Mar 2019 05:14:39 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy2.7-v7.x: start release branch Message-ID: <5c86512f.1c69fb81.cda7a.1688@mx.google.com> Author: Matti Picus Branch: release-pypy2.7-v7.x Changeset: r96271:4d17db2673b9 Date: 2019-03-11 13:18 +0200 http://bitbucket.org/pypy/pypy/changeset/4d17db2673b9/ Log: start release branch diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -32,7 +32,7 @@ * module/sys/version.py * doc/conf.py */ -#define PYPY_VERSION "7.1.0-alpha0" +#define PYPY_VERSION "7.1.0" #define PYPY_VERSION_NUM 0x07010000 /* Defined to mean a PyPy where cpyext holds more regular references to PyObjects, e.g. staying alive as long as the internal PyPy object diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -13,7 +13,7 @@ # make sure to keep PYPY_VERSION in sync with: # module/cpyext/include/patchlevel.h # doc/conf.py -PYPY_VERSION = (7, 1, 0, "alpha", 0) +PYPY_VERSION = (7, 1, 0, "final", 0) import pypy From pypy.commits at gmail.com Mon Mar 11 08:14:42 2019 From: pypy.commits at gmail.com (mattip) Date: Mon, 11 Mar 2019 05:14:42 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy3.6-v7.x: start release branch Message-ID: <5c865132.1c69fb81.6b482.85ef@mx.google.com> Author: Matti Picus Branch: release-pypy3.6-v7.x Changeset: r96272:a6406bc3dda2 Date: 2019-03-11 13:18 +0200 http://bitbucket.org/pypy/pypy/changeset/a6406bc3dda2/ Log: start release branch diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -32,7 +32,7 @@ * module/sys/version.py * doc/conf.py */ -#define PYPY_VERSION "7.1.0-alpha0" +#define PYPY_VERSION "7.1.0" #define PYPY_VERSION_NUM 0x07010000 /* Defined to mean a PyPy where cpyext holds more regular references to PyObjects, e.g. staying alive as long as the internal PyPy object diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -13,7 +13,7 @@ # make sure to keep PYPY_VERSION in sync with: # module/cpyext/include/patchlevel.h # doc/conf.py -PYPY_VERSION = (7, 1, 0, "alpha", 0) +PYPY_VERSION = (7, 1, 0, "final", 0) import pypy From pypy.commits at gmail.com Mon Mar 11 08:14:43 2019 From: pypy.commits at gmail.com (mattip) Date: Mon, 11 Mar 2019 05:14:43 -0700 (PDT) Subject: [pypy-commit] pypy default: update version, move whatsnew-head Message-ID: <5c865133.1c69fb81.affd2.0dcd@mx.google.com> Author: Matti Picus Branch: Changeset: r96273:78914a03cf95 Date: 2019-03-11 13:23 +0200 http://bitbucket.org/pypy/pypy/changeset/78914a03cf95/ Log: update version, move whatsnew-head diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -71,9 +71,9 @@ # module/cpyext/include/patchlevel.h # # The short X.Y version. -version = '7.1' +version = '7.2' # The full version, including alpha/beta/rc tags. -release = '7.1.0' +release = '7.2.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-7.1.0.rst rename from pypy/doc/whatsnew-head.rst rename to pypy/doc/whatsnew-pypy2-7.1.0.rst diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -32,8 +32,8 @@ * module/sys/version.py * doc/conf.py */ -#define PYPY_VERSION "7.1.0-alpha0" -#define PYPY_VERSION_NUM 0x07010000 +#define PYPY_VERSION "7.2.0-alpha0" +#define PYPY_VERSION_NUM 0x07020000 /* Defined to mean a PyPy where cpyext holds more regular references to PyObjects, e.g. staying alive as long as the internal PyPy object stays alive. */ diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -13,7 +13,7 @@ # make sure to keep PYPY_VERSION in sync with: # module/cpyext/include/patchlevel.h # doc/conf.py -PYPY_VERSION = (7, 1, 0, "alpha", 0) +PYPY_VERSION = (7, 2, 0, "alpha", 0) import pypy From pypy.commits at gmail.com Mon Mar 11 08:14:45 2019 From: pypy.commits at gmail.com (mattip) Date: Mon, 11 Mar 2019 05:14:45 -0700 (PDT) Subject: [pypy-commit] pypy default: restart whatsnew-head Message-ID: <5c865135.1c69fb81.505be.bcfc@mx.google.com> Author: Matti Picus Branch: Changeset: r96274:cf8eb4195235 Date: 2019-03-11 13:24 +0200 http://bitbucket.org/pypy/pypy/changeset/cf8eb4195235/ Log: restart whatsnew-head diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-head.rst @@ -0,0 +1,7 @@ +========================== +What's new in PyPy2.7 7.1+ +========================== + +.. this is a revision shortly after release-pypy-7.1.0 +.. startrev: 78914a03cf95 + From pypy.commits at gmail.com Mon Mar 11 08:14:47 2019 From: pypy.commits at gmail.com (mattip) Date: Mon, 11 Mar 2019 05:14:47 -0700 (PDT) Subject: [pypy-commit] pypy default: start release note Message-ID: <5c865137.1c69fb81.f8ed2.4386@mx.google.com> Author: Matti Picus Branch: Changeset: r96275:ea739add7f2a Date: 2019-03-11 13:31 +0200 http://bitbucket.org/pypy/pypy/changeset/ea739add7f2a/ Log: start release note diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -1,11 +1,12 @@ Historical release notes ======================== -CPython 2.7 compatible versions -------------------------------- +Combined releases +----------------- .. toctree:: + release-v7.1.0.rst release-v7.0.0.rst release-v6.0.0.rst release-v5.10.1.rst @@ -14,6 +15,12 @@ release-v5.8.0.rst release-v5.7.1.rst release-v5.7.0.rst + +CPython 2.7 compatible versions +------------------------------- + +.. toctree:: + release-pypy2.7-v5.6.0.rst release-pypy2.7-v5.4.1.rst release-pypy2.7-v5.4.0.rst @@ -61,15 +68,6 @@ release-0.7.0.rst release-0.6 -CPython 3.5 compatible versions -------------------------------- - -.. toctree:: - - release-v5.8.0.rst - release-v5.7.1.rst - release-v5.7.0.rst - CPython 3.3 compatible versions ------------------------------- diff --git a/pypy/doc/release-v7.1.0.rst b/pypy/doc/release-v7.1.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v7.1.0.rst @@ -0,0 +1,77 @@ +========================================= +PyPy v7.1.0: release of 2.7, and 3.6-beta +========================================= + +The PyPy team is proud to release the version 7.1.0 of PyPy, which includes +two different interpreters: + + - PyPy2.7, which is an interpreter supporting the syntax and the features of + Python 2.7 + + - PyPy3.6-beta: this is the second official release of PyPy to support 3.6 + features, although it is still considered beta quality. + +The interpreters are based on much the same codebase, thus the double +release. + +Until we can work with downstream providers to distribute builds with PyPy, we +have made packages for some common packages `available as wheels`_. + +As always, this release is 100% compatible with the previous one and fixed +several issues and bugs raised by the growing community of PyPy users. +We strongly recommend updating. + +The PyPy3.6 release is still not production quality so your mileage may vary. +There are open issues with incomplete compatibility and c-extension support. + +You can download the v7.0 releases here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. If PyPy is not quite good enough for your needs, we are available for +direct consulting work. + +We would also like to thank our contributors and encourage new people to join +the project. PyPy has many layers and we need help with all of them: `PyPy`_ +and `RPython`_ documentation improvements, tweaking popular modules to run +on pypy, or general `help`_ with making RPython's JIT even better. + +.. _`PyPy`: index.html +.. _`RPython`: https://rpython.readthedocs.org +.. _`help`: project-ideas.html +.. _`cffi`: http://cffi.readthedocs.io +.. _`cppyy`: https://cppyy.readthedocs.io +.. _`available as wheels`: https://github.com/antocuni/pypy-wheels + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7, 3.6. It's fast (`PyPy and CPython 2.7.x`_ performance +comparison) due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +The PyPy release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +Unfortunately at the moment of writing our ARM buildbots are out of service, +so for now we are **not** releasing any binary for the ARM architecture. + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + + +Changelog +========= + +If not specified, the changes are shared across versions + From pypy.commits at gmail.com Mon Mar 11 08:14:48 2019 From: pypy.commits at gmail.com (mattip) Date: Mon, 11 Mar 2019 05:14:48 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: merge default into py3.6 Message-ID: <5c865138.1c69fb81.12a8d.a6a0@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r96276:d642a3c217cb Date: 2019-03-11 13:58 +0200 http://bitbucket.org/pypy/pypy/changeset/d642a3c217cb/ Log: merge default into py3.6 diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py --- a/lib_pypy/_ctypes/array.py +++ b/lib_pypy/_ctypes/array.py @@ -263,7 +263,7 @@ try: itemsize = struct.calcsize(fmt[1:]) except: - itemsize = len(buffer(obj[0])) + itemsize = sizeof(obj[0]) return __pypy__.newmemoryview(memoryview(self._buffer), itemsize, fmt, shape) ARRAY_CACHE = {} diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -71,9 +71,9 @@ # module/cpyext/include/patchlevel.h # # The short X.Y version. -version = '7.1' +version = '7.2' # The full version, including alpha/beta/rc tags. -release = '7.1.0' +release = '7.2.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -1,11 +1,12 @@ Historical release notes ======================== -CPython 2.7 compatible versions -------------------------------- +Combined releases +----------------- .. toctree:: + release-v7.1.0.rst release-v7.0.0.rst release-v6.0.0.rst release-v5.10.1.rst @@ -14,6 +15,12 @@ release-v5.8.0.rst release-v5.7.1.rst release-v5.7.0.rst + +CPython 2.7 compatible versions +------------------------------- + +.. toctree:: + release-pypy2.7-v5.6.0.rst release-pypy2.7-v5.4.1.rst release-pypy2.7-v5.4.0.rst @@ -61,15 +68,6 @@ release-0.7.0.rst release-0.6 -CPython 3.5 compatible versions -------------------------------- - -.. toctree:: - - release-v5.8.0.rst - release-v5.7.1.rst - release-v5.7.0.rst - CPython 3.3 compatible versions ------------------------------- diff --git a/pypy/doc/release-v7.1.0.rst b/pypy/doc/release-v7.1.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v7.1.0.rst @@ -0,0 +1,77 @@ +========================================= +PyPy v7.1.0: release of 2.7, and 3.6-beta +========================================= + +The PyPy team is proud to release the version 7.1.0 of PyPy, which includes +two different interpreters: + + - PyPy2.7, which is an interpreter supporting the syntax and the features of + Python 2.7 + + - PyPy3.6-beta: this is the second official release of PyPy to support 3.6 + features, although it is still considered beta quality. + +The interpreters are based on much the same codebase, thus the double +release. + +Until we can work with downstream providers to distribute builds with PyPy, we +have made packages for some common packages `available as wheels`_. + +As always, this release is 100% compatible with the previous one and fixed +several issues and bugs raised by the growing community of PyPy users. +We strongly recommend updating. + +The PyPy3.6 release is still not production quality so your mileage may vary. +There are open issues with incomplete compatibility and c-extension support. + +You can download the v7.0 releases here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. If PyPy is not quite good enough for your needs, we are available for +direct consulting work. + +We would also like to thank our contributors and encourage new people to join +the project. PyPy has many layers and we need help with all of them: `PyPy`_ +and `RPython`_ documentation improvements, tweaking popular modules to run +on pypy, or general `help`_ with making RPython's JIT even better. + +.. _`PyPy`: index.html +.. _`RPython`: https://rpython.readthedocs.org +.. _`help`: project-ideas.html +.. _`cffi`: http://cffi.readthedocs.io +.. _`cppyy`: https://cppyy.readthedocs.io +.. _`available as wheels`: https://github.com/antocuni/pypy-wheels + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7, 3.6. It's fast (`PyPy and CPython 2.7.x`_ performance +comparison) due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +The PyPy release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +Unfortunately at the moment of writing our ARM buildbots are out of service, +so for now we are **not** releasing any binary for the ARM architecture. + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + + +Changelog +========= + +If not specified, the changes are shared across versions + diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,41 +1,6 @@ ========================== -What's new in PyPy2.7 7.0+ +What's new in PyPy2.7 7.1+ ========================== -.. this is a revision shortly after release-pypy-7.0.0 -.. startrev: 481c69f7d81f - -.. branch: zlib-copying-third-time-a-charm - -Make sure zlib decompressobjs have their streams deallocated immediately -on flush. - -.. branch: zlib-copying-redux - -Fix calling copy on already-flushed compressobjs. - - - -.. branch: math-improvements - -Improve performance of long operations where one of the operands fits into -an int. - -.. branch: regalloc-playground - -Improve register allocation in the JIT. - -.. branch: promote-unicode - -Implement rlib.jit.promote_unicode to complement promote_string - -.. branch: unicode-utf8 - -Use utf8 internally to represent unicode, with the goal of never using rpython-level unicode - -.. branch: newmemoryview-app-level - -Since _ctypes is implemented in pure python over libffi, add interfaces and -methods to support the buffer interface from python. Specifically, add a -``__pypy__.newmemoryview`` function to create a memoryview and extend the use -of the PyPy-specific ``__buffer__`` class method. \ No newline at end of file +.. this is a revision shortly after release-pypy-7.1.0 +.. startrev: 78914a03cf95 diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-7.1.0.rst copy from pypy/doc/whatsnew-head.rst copy to pypy/doc/whatsnew-pypy2-7.1.0.rst diff --git a/pypy/module/__pypy__/test/test_newmemoryview.py b/pypy/module/__pypy__/test/test_newmemoryview.py --- a/pypy/module/__pypy__/test/test_newmemoryview.py +++ b/pypy/module/__pypy__/test/test_newmemoryview.py @@ -19,7 +19,7 @@ from __pypy__ import bufferable, newmemoryview class B(bufferable.bufferable): def __init__(self): - self.data = bytearray('abc') + self.data = bytearray(b'abc') def __buffer__(self, flags): return newmemoryview(memoryview(self.data), 1, 'B') diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py --- a/pypy/module/_cffi_backend/ctypestruct.py +++ b/pypy/module/_cffi_backend/ctypestruct.py @@ -238,26 +238,32 @@ else: self.ctype.convert_from_object(cdata, w_ob) + def add_varsize_length(self, space, itemsize, varsizelength, optvarsize): + # returns an updated 'optvarsize' to account for an array of + # 'varsizelength' elements, each of size 'itemsize', that starts + # at 'self.offset'. + try: + varsize = ovfcheck(itemsize * varsizelength) + size = ovfcheck(self.offset + varsize) + except OverflowError: + raise oefmt(space.w_OverflowError, + "array size would overflow a ssize_t") + assert size >= 0 + return max(size, optvarsize) + def write_v(self, cdata, w_ob, optvarsize): # a special case for var-sized C99 arrays from pypy.module._cffi_backend import ctypearray ct = self.ctype + space = ct.space if isinstance(ct, ctypearray.W_CTypeArray) and ct.length < 0: - space = ct.space w_ob, varsizelength = ct.get_new_array_length(w_ob) if optvarsize != -1: # in this mode, the only purpose of this function is to compute # the real size of the structure from a var-sized C99 array assert cdata == lltype.nullptr(rffi.CCHARP.TO) - itemsize = ct.ctitem.size - try: - varsize = ovfcheck(itemsize * varsizelength) - size = ovfcheck(self.offset + varsize) - except OverflowError: - raise oefmt(space.w_OverflowError, - "array size would overflow a ssize_t") - assert size >= 0 - return max(size, optvarsize) + return self.add_varsize_length(space, ct.ctitem.size, + varsizelength, optvarsize) # if 'value' was only an integer, get_new_array_length() returns # w_ob = space.w_None. Detect if this was the case, # and if so, stop here, leaving the content uninitialized @@ -267,6 +273,12 @@ # if optvarsize == -1: self.write(cdata, w_ob) + elif (isinstance(ct, W_CTypeStructOrUnion) and ct._with_var_array and + not isinstance(w_ob, cdataobj.W_CData)): + subsize = ct.size + subsize = ct.convert_struct_from_object( + lltype.nullptr(rffi.CCHARP.TO), w_ob, subsize) + optvarsize = self.add_varsize_length(space, 1, subsize, optvarsize) return optvarsize def convert_bitfield_to_object(self, cdata): diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py --- a/pypy/module/_cffi_backend/newtype.py +++ b/pypy/module/_cffi_backend/newtype.py @@ -368,6 +368,16 @@ raise oefmt(space.w_TypeError, "field '%s.%s' has ctype '%s' of unknown size", w_ctype.name, fname, ftype.name) + elif isinstance(ftype, ctypestruct.W_CTypeStructOrUnion): + ftype.force_lazy_struct() + # GCC (or maybe C99) accepts var-sized struct fields that are not + # the last field of a larger struct. That's why there is no + # check here for "last field": we propagate the flag + # '_with_var_array' to any struct that contains either an open- + # ended array or another struct that recursively contains an + # open-ended array. + if ftype._with_var_array: + with_var_array = True # if is_union: boffset = 0 # reset each field at offset 0 @@ -419,7 +429,6 @@ # a nested anonymous struct or union # note: it seems we only get here with ffi.verify() srcfield2names = {} - ftype.force_lazy_struct() for name, srcfld in ftype._fields_dict.items(): srcfield2names[srcfld] = name for srcfld in ftype._fields_list: diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -3441,6 +3441,15 @@ assert p.a[1] == 20 assert p.a[2] == 30 assert p.a[3] == 0 + # + # struct of struct of varsized array + BStruct2 = new_struct_type("bar") + complete_struct_or_union(BStruct2, [('head', BInt), + ('tail', BStruct)]) + for i in range(2): # try to detect heap overwrites + p = newp(new_pointer_type(BStruct2), [100, [200, list(range(50))]]) + assert p.tail.y[49] == 49 + def test_struct_array_no_length_explicit_position(): BInt = new_primitive_type("int") diff --git a/pypy/module/_multiprocessing/test/test_win32.py b/pypy/module/_multiprocessing/test/test_win32.py --- a/pypy/module/_multiprocessing/test/test_win32.py +++ b/pypy/module/_multiprocessing/test/test_win32.py @@ -2,7 +2,7 @@ import sys class AppTestWin32: - spaceconfig = dict(usemodules=('_multiprocessing', _cffi_backend', + spaceconfig = dict(usemodules=('_multiprocessing', '_cffi_backend', 'signal', '_rawffi', 'binascii')) def setup_class(cls): diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -32,8 +32,8 @@ * module/sys/version.py * doc/conf.py */ -#define PYPY_VERSION "7.1.0-alpha0" -#define PYPY_VERSION_NUM 0x07010000 +#define PYPY_VERSION "7.2.0-alpha0" +#define PYPY_VERSION_NUM 0x07020000 /* Defined to mean a PyPy where cpyext holds more regular references to PyObjects, e.g. staying alive as long as the internal PyPy object stays alive. */ diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -13,7 +13,7 @@ # make sure to keep PYPY_VERSION in sync with: # module/cpyext/include/patchlevel.h # doc/conf.py -PYPY_VERSION = (7, 1, 0, "alpha", 0) +PYPY_VERSION = (7, 2, 0, "alpha", 0) import pypy From pypy.commits at gmail.com Mon Mar 11 08:14:50 2019 From: pypy.commits at gmail.com (mattip) Date: Mon, 11 Mar 2019 05:14:50 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: move whatsnew Message-ID: <5c86513a.1c69fb81.981a0.77c1@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r96277:92e8cf22beda Date: 2019-03-11 14:01 +0200 http://bitbucket.org/pypy/pypy/changeset/92e8cf22beda/ Log: move whatsnew diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-7.1.0.rst rename from pypy/doc/whatsnew-pypy3-head.rst rename to pypy/doc/whatsnew-pypy3-7.1.0.rst From pypy.commits at gmail.com Mon Mar 11 08:14:52 2019 From: pypy.commits at gmail.com (mattip) Date: Mon, 11 Mar 2019 05:14:52 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: restart whatsnew Message-ID: <5c86513c.1c69fb81.182d7.74fd@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r96278:aafbac9cc1f8 Date: 2019-03-11 14:02 +0200 http://bitbucket.org/pypy/pypy/changeset/aafbac9cc1f8/ Log: restart whatsnew diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy3-head.rst @@ -0,0 +1,6 @@ +======================== +What's new in PyPy3 7.1+ +======================== + +.. this is the revision after release-pypy3.6-v7.1 +.. startrev: d642a3c217cb From pypy.commits at gmail.com Mon Mar 11 08:14:54 2019 From: pypy.commits at gmail.com (mattip) Date: Mon, 11 Mar 2019 05:14:54 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy3.6-v7.x: merge py3.6 into release (preserve versioning) Message-ID: <5c86513e.1c69fb81.2098e.243d@mx.google.com> Author: Matti Picus Branch: release-pypy3.6-v7.x Changeset: r96279:a05759e2b374 Date: 2019-03-11 14:02 +0200 http://bitbucket.org/pypy/pypy/changeset/a05759e2b374/ Log: merge py3.6 into release (preserve versioning) diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py --- a/lib_pypy/_ctypes/array.py +++ b/lib_pypy/_ctypes/array.py @@ -263,7 +263,7 @@ try: itemsize = struct.calcsize(fmt[1:]) except: - itemsize = len(buffer(obj[0])) + itemsize = sizeof(obj[0]) return __pypy__.newmemoryview(memoryview(self._buffer), itemsize, fmt, shape) ARRAY_CACHE = {} diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -71,9 +71,9 @@ # module/cpyext/include/patchlevel.h # # The short X.Y version. -version = '7.1' +version = '7.2' # The full version, including alpha/beta/rc tags. -release = '7.1.0' +release = '7.2.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -1,11 +1,12 @@ Historical release notes ======================== -CPython 2.7 compatible versions -------------------------------- +Combined releases +----------------- .. toctree:: + release-v7.1.0.rst release-v7.0.0.rst release-v6.0.0.rst release-v5.10.1.rst @@ -14,6 +15,12 @@ release-v5.8.0.rst release-v5.7.1.rst release-v5.7.0.rst + +CPython 2.7 compatible versions +------------------------------- + +.. toctree:: + release-pypy2.7-v5.6.0.rst release-pypy2.7-v5.4.1.rst release-pypy2.7-v5.4.0.rst @@ -61,15 +68,6 @@ release-0.7.0.rst release-0.6 -CPython 3.5 compatible versions -------------------------------- - -.. toctree:: - - release-v5.8.0.rst - release-v5.7.1.rst - release-v5.7.0.rst - CPython 3.3 compatible versions ------------------------------- diff --git a/pypy/doc/release-v7.1.0.rst b/pypy/doc/release-v7.1.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v7.1.0.rst @@ -0,0 +1,77 @@ +========================================= +PyPy v7.1.0: release of 2.7, and 3.6-beta +========================================= + +The PyPy team is proud to release the version 7.1.0 of PyPy, which includes +two different interpreters: + + - PyPy2.7, which is an interpreter supporting the syntax and the features of + Python 2.7 + + - PyPy3.6-beta: this is the second official release of PyPy to support 3.6 + features, although it is still considered beta quality. + +The interpreters are based on much the same codebase, thus the double +release. + +Until we can work with downstream providers to distribute builds with PyPy, we +have made packages for some common packages `available as wheels`_. + +As always, this release is 100% compatible with the previous one and fixed +several issues and bugs raised by the growing community of PyPy users. +We strongly recommend updating. + +The PyPy3.6 release is still not production quality so your mileage may vary. +There are open issues with incomplete compatibility and c-extension support. + +You can download the v7.0 releases here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. If PyPy is not quite good enough for your needs, we are available for +direct consulting work. + +We would also like to thank our contributors and encourage new people to join +the project. PyPy has many layers and we need help with all of them: `PyPy`_ +and `RPython`_ documentation improvements, tweaking popular modules to run +on pypy, or general `help`_ with making RPython's JIT even better. + +.. _`PyPy`: index.html +.. _`RPython`: https://rpython.readthedocs.org +.. _`help`: project-ideas.html +.. _`cffi`: http://cffi.readthedocs.io +.. _`cppyy`: https://cppyy.readthedocs.io +.. _`available as wheels`: https://github.com/antocuni/pypy-wheels + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7, 3.6. It's fast (`PyPy and CPython 2.7.x`_ performance +comparison) due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +The PyPy release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +Unfortunately at the moment of writing our ARM buildbots are out of service, +so for now we are **not** releasing any binary for the ARM architecture. + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + + +Changelog +========= + +If not specified, the changes are shared across versions + diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,41 +1,6 @@ ========================== -What's new in PyPy2.7 7.0+ +What's new in PyPy2.7 7.1+ ========================== -.. this is a revision shortly after release-pypy-7.0.0 -.. startrev: 481c69f7d81f - -.. branch: zlib-copying-third-time-a-charm - -Make sure zlib decompressobjs have their streams deallocated immediately -on flush. - -.. branch: zlib-copying-redux - -Fix calling copy on already-flushed compressobjs. - - - -.. branch: math-improvements - -Improve performance of long operations where one of the operands fits into -an int. - -.. branch: regalloc-playground - -Improve register allocation in the JIT. - -.. branch: promote-unicode - -Implement rlib.jit.promote_unicode to complement promote_string - -.. branch: unicode-utf8 - -Use utf8 internally to represent unicode, with the goal of never using rpython-level unicode - -.. branch: newmemoryview-app-level - -Since _ctypes is implemented in pure python over libffi, add interfaces and -methods to support the buffer interface from python. Specifically, add a -``__pypy__.newmemoryview`` function to create a memoryview and extend the use -of the PyPy-specific ``__buffer__`` class method. \ No newline at end of file +.. this is a revision shortly after release-pypy-7.1.0 +.. startrev: 78914a03cf95 diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-7.1.0.rst copy from pypy/doc/whatsnew-head.rst copy to pypy/doc/whatsnew-pypy2-7.1.0.rst diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-7.1.0.rst copy from pypy/doc/whatsnew-pypy3-head.rst copy to pypy/doc/whatsnew-pypy3-7.1.0.rst diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst --- a/pypy/doc/whatsnew-pypy3-head.rst +++ b/pypy/doc/whatsnew-pypy3-head.rst @@ -1,11 +1,6 @@ ======================== -What's new in PyPy3 7.0+ +What's new in PyPy3 7.1+ ======================== -.. this is the revision after release-pypy3.6-v7.0 -.. startrev: 33fe3b2cf186 - -.. branch: py3.5 - -Merge in py.35 and use this branch as the primary pypy3 one - +.. this is the revision after release-pypy3.6-v7.1 +.. startrev: d642a3c217cb diff --git a/pypy/module/__pypy__/test/test_newmemoryview.py b/pypy/module/__pypy__/test/test_newmemoryview.py --- a/pypy/module/__pypy__/test/test_newmemoryview.py +++ b/pypy/module/__pypy__/test/test_newmemoryview.py @@ -19,7 +19,7 @@ from __pypy__ import bufferable, newmemoryview class B(bufferable.bufferable): def __init__(self): - self.data = bytearray('abc') + self.data = bytearray(b'abc') def __buffer__(self, flags): return newmemoryview(memoryview(self.data), 1, 'B') diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py --- a/pypy/module/_cffi_backend/ctypestruct.py +++ b/pypy/module/_cffi_backend/ctypestruct.py @@ -238,26 +238,32 @@ else: self.ctype.convert_from_object(cdata, w_ob) + def add_varsize_length(self, space, itemsize, varsizelength, optvarsize): + # returns an updated 'optvarsize' to account for an array of + # 'varsizelength' elements, each of size 'itemsize', that starts + # at 'self.offset'. + try: + varsize = ovfcheck(itemsize * varsizelength) + size = ovfcheck(self.offset + varsize) + except OverflowError: + raise oefmt(space.w_OverflowError, + "array size would overflow a ssize_t") + assert size >= 0 + return max(size, optvarsize) + def write_v(self, cdata, w_ob, optvarsize): # a special case for var-sized C99 arrays from pypy.module._cffi_backend import ctypearray ct = self.ctype + space = ct.space if isinstance(ct, ctypearray.W_CTypeArray) and ct.length < 0: - space = ct.space w_ob, varsizelength = ct.get_new_array_length(w_ob) if optvarsize != -1: # in this mode, the only purpose of this function is to compute # the real size of the structure from a var-sized C99 array assert cdata == lltype.nullptr(rffi.CCHARP.TO) - itemsize = ct.ctitem.size - try: - varsize = ovfcheck(itemsize * varsizelength) - size = ovfcheck(self.offset + varsize) - except OverflowError: - raise oefmt(space.w_OverflowError, - "array size would overflow a ssize_t") - assert size >= 0 - return max(size, optvarsize) + return self.add_varsize_length(space, ct.ctitem.size, + varsizelength, optvarsize) # if 'value' was only an integer, get_new_array_length() returns # w_ob = space.w_None. Detect if this was the case, # and if so, stop here, leaving the content uninitialized @@ -267,6 +273,12 @@ # if optvarsize == -1: self.write(cdata, w_ob) + elif (isinstance(ct, W_CTypeStructOrUnion) and ct._with_var_array and + not isinstance(w_ob, cdataobj.W_CData)): + subsize = ct.size + subsize = ct.convert_struct_from_object( + lltype.nullptr(rffi.CCHARP.TO), w_ob, subsize) + optvarsize = self.add_varsize_length(space, 1, subsize, optvarsize) return optvarsize def convert_bitfield_to_object(self, cdata): diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py --- a/pypy/module/_cffi_backend/newtype.py +++ b/pypy/module/_cffi_backend/newtype.py @@ -368,6 +368,16 @@ raise oefmt(space.w_TypeError, "field '%s.%s' has ctype '%s' of unknown size", w_ctype.name, fname, ftype.name) + elif isinstance(ftype, ctypestruct.W_CTypeStructOrUnion): + ftype.force_lazy_struct() + # GCC (or maybe C99) accepts var-sized struct fields that are not + # the last field of a larger struct. That's why there is no + # check here for "last field": we propagate the flag + # '_with_var_array' to any struct that contains either an open- + # ended array or another struct that recursively contains an + # open-ended array. + if ftype._with_var_array: + with_var_array = True # if is_union: boffset = 0 # reset each field at offset 0 @@ -419,7 +429,6 @@ # a nested anonymous struct or union # note: it seems we only get here with ffi.verify() srcfield2names = {} - ftype.force_lazy_struct() for name, srcfld in ftype._fields_dict.items(): srcfield2names[srcfld] = name for srcfld in ftype._fields_list: diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -3441,6 +3441,15 @@ assert p.a[1] == 20 assert p.a[2] == 30 assert p.a[3] == 0 + # + # struct of struct of varsized array + BStruct2 = new_struct_type("bar") + complete_struct_or_union(BStruct2, [('head', BInt), + ('tail', BStruct)]) + for i in range(2): # try to detect heap overwrites + p = newp(new_pointer_type(BStruct2), [100, [200, list(range(50))]]) + assert p.tail.y[49] == 49 + def test_struct_array_no_length_explicit_position(): BInt = new_primitive_type("int") diff --git a/pypy/module/_multiprocessing/test/test_win32.py b/pypy/module/_multiprocessing/test/test_win32.py --- a/pypy/module/_multiprocessing/test/test_win32.py +++ b/pypy/module/_multiprocessing/test/test_win32.py @@ -2,7 +2,7 @@ import sys class AppTestWin32: - spaceconfig = dict(usemodules=('_multiprocessing', _cffi_backend', + spaceconfig = dict(usemodules=('_multiprocessing', '_cffi_backend', 'signal', '_rawffi', 'binascii')) def setup_class(cls): From pypy.commits at gmail.com Mon Mar 11 08:14:55 2019 From: pypy.commits at gmail.com (mattip) Date: Mon, 11 Mar 2019 05:14:55 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy2.7-v7.x: merge default into release (perserve versioning) Message-ID: <5c86513f.1c69fb81.18223.781e@mx.google.com> Author: Matti Picus Branch: release-pypy2.7-v7.x Changeset: r96280:3a2619127f7e Date: 2019-03-11 14:03 +0200 http://bitbucket.org/pypy/pypy/changeset/3a2619127f7e/ Log: merge default into release (perserve versioning) diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -71,9 +71,9 @@ # module/cpyext/include/patchlevel.h # # The short X.Y version. -version = '7.1' +version = '7.2' # The full version, including alpha/beta/rc tags. -release = '7.1.0' +release = '7.2.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -1,11 +1,12 @@ Historical release notes ======================== -CPython 2.7 compatible versions -------------------------------- +Combined releases +----------------- .. toctree:: + release-v7.1.0.rst release-v7.0.0.rst release-v6.0.0.rst release-v5.10.1.rst @@ -14,6 +15,12 @@ release-v5.8.0.rst release-v5.7.1.rst release-v5.7.0.rst + +CPython 2.7 compatible versions +------------------------------- + +.. toctree:: + release-pypy2.7-v5.6.0.rst release-pypy2.7-v5.4.1.rst release-pypy2.7-v5.4.0.rst @@ -61,15 +68,6 @@ release-0.7.0.rst release-0.6 -CPython 3.5 compatible versions -------------------------------- - -.. toctree:: - - release-v5.8.0.rst - release-v5.7.1.rst - release-v5.7.0.rst - CPython 3.3 compatible versions ------------------------------- diff --git a/pypy/doc/release-v7.1.0.rst b/pypy/doc/release-v7.1.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v7.1.0.rst @@ -0,0 +1,77 @@ +========================================= +PyPy v7.1.0: release of 2.7, and 3.6-beta +========================================= + +The PyPy team is proud to release the version 7.1.0 of PyPy, which includes +two different interpreters: + + - PyPy2.7, which is an interpreter supporting the syntax and the features of + Python 2.7 + + - PyPy3.6-beta: this is the second official release of PyPy to support 3.6 + features, although it is still considered beta quality. + +The interpreters are based on much the same codebase, thus the double +release. + +Until we can work with downstream providers to distribute builds with PyPy, we +have made packages for some common packages `available as wheels`_. + +As always, this release is 100% compatible with the previous one and fixed +several issues and bugs raised by the growing community of PyPy users. +We strongly recommend updating. + +The PyPy3.6 release is still not production quality so your mileage may vary. +There are open issues with incomplete compatibility and c-extension support. + +You can download the v7.0 releases here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. If PyPy is not quite good enough for your needs, we are available for +direct consulting work. + +We would also like to thank our contributors and encourage new people to join +the project. PyPy has many layers and we need help with all of them: `PyPy`_ +and `RPython`_ documentation improvements, tweaking popular modules to run +on pypy, or general `help`_ with making RPython's JIT even better. + +.. _`PyPy`: index.html +.. _`RPython`: https://rpython.readthedocs.org +.. _`help`: project-ideas.html +.. _`cffi`: http://cffi.readthedocs.io +.. _`cppyy`: https://cppyy.readthedocs.io +.. _`available as wheels`: https://github.com/antocuni/pypy-wheels + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7, 3.6. It's fast (`PyPy and CPython 2.7.x`_ performance +comparison) due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +The PyPy release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +Unfortunately at the moment of writing our ARM buildbots are out of service, +so for now we are **not** releasing any binary for the ARM architecture. + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + + +Changelog +========= + +If not specified, the changes are shared across versions + diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,45 +1,7 @@ ========================== -What's new in PyPy2.7 7.0+ +What's new in PyPy2.7 7.1+ ========================== -.. this is a revision shortly after release-pypy-7.0.0 -.. startrev: 481c69f7d81f +.. this is a revision shortly after release-pypy-7.1.0 +.. startrev: 78914a03cf95 -.. branch: zlib-copying-third-time-a-charm - -Make sure zlib decompressobjs have their streams deallocated immediately -on flush. - -.. branch: zlib-copying-redux - -Fix calling copy on already-flushed compressobjs. - -.. branch: zlib-copying - -The zlib module's compressobj and decompressobj now expose copy methods -as they do on CPython. - - -.. branch: math-improvements - -Improve performance of long operations where one of the operands fits into -an int. - -.. branch: regalloc-playground - -Improve register allocation in the JIT. - -.. branch: promote-unicode - -Implement rlib.jit.promote_unicode to complement promote_string - -.. branch: unicode-utf8 - -Use utf8 internally to represent unicode, with the goal of never using rpython-level unicode - -.. branch: newmemoryview-app-level - -Since _ctypes is implemented in pure python over libffi, add interfaces and -methods to support the buffer interface from python. Specifically, add a -``__pypy__.newmemoryview`` function to create a memoryview and extend the use -of the PyPy-specific ``__buffer__`` class method. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-7.1.0.rst copy from pypy/doc/whatsnew-head.rst copy to pypy/doc/whatsnew-pypy2-7.1.0.rst From pypy.commits at gmail.com Mon Mar 11 08:14:57 2019 From: pypy.commits at gmail.com (mattip) Date: Mon, 11 Mar 2019 05:14:57 -0700 (PDT) Subject: [pypy-commit] pypy default: extend release note Message-ID: <5c865141.1c69fb81.1d92a.1620@mx.google.com> Author: Matti Picus Branch: Changeset: r96281:20d6b720c743 Date: 2019-03-11 14:12 +0200 http://bitbucket.org/pypy/pypy/changeset/20d6b720c743/ Log: extend release note diff --git a/pypy/doc/release-v7.1.0.rst b/pypy/doc/release-v7.1.0.rst --- a/pypy/doc/release-v7.1.0.rst +++ b/pypy/doc/release-v7.1.0.rst @@ -14,6 +14,13 @@ The interpreters are based on much the same codebase, thus the double release. +This release, coming fast on the heels of 7.0 in February, finally merges the +internal refactoring of unicode representation as UTF-8. Removing the +conversions from strings to unicode internally lead to a nice speed bump. + +We also improved the ability to use the buffer protocol with ctype structures +and arrays. + Until we can work with downstream providers to distribute builds with PyPy, we have made packages for some common packages `available as wheels`_. From pypy.commits at gmail.com Mon Mar 11 13:42:38 2019 From: pypy.commits at gmail.com (mattip) Date: Mon, 11 Mar 2019 10:42:38 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy3.6-v7.x: merge py3.6 into release Message-ID: <5c869e0e.1c69fb81.d7b4.1dce@mx.google.com> Author: Matti Picus Branch: release-pypy3.6-v7.x Changeset: r96285:5e9a4737b5a7 Date: 2019-03-11 19:41 +0200 http://bitbucket.org/pypy/pypy/changeset/5e9a4737b5a7/ Log: merge py3.6 into release diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -123,7 +123,9 @@ Wenzhu Man Konstantin Lopuhin John Witulski + Stefan Beyer Jeremy Thurgood + Andrew Lawrence Greg Price Ivan Sichmann Freitas Dario Bertini @@ -134,7 +136,6 @@ Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov - Stefan Beyer William Leslie Paweł Piotr Przeradowski marky1991 @@ -152,6 +153,7 @@ Wanja Saatkamp Mike Blume Gerald Klix + Julian Berman Oscar Nierstrasz Rami Chowdhury Stefan H. Muller @@ -174,6 +176,7 @@ Anton Gulenko Sergey Matyunin Andrew Chambers + Łukasz Langa Nicolas Chauvat Andrew Durdin Ben Young @@ -296,7 +299,6 @@ Bobby Impollonia Roberto De Ioris Jeong YunWon - andrewjlawrence Christopher Armstrong Aaron Tubbs Vasantha Ganesh K @@ -328,7 +330,6 @@ Ben Darnell Juan Francisco Cantero Hurtado Godefroid Chappelle - Julian Berman Stephan Busemann Dan Colish timo diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -90,7 +90,9 @@ Wenzhu Man Konstantin Lopuhin John Witulski + Stefan Beyer Jeremy Thurgood + Andrew Lawrence Greg Price Ivan Sichmann Freitas Dario Bertini @@ -101,7 +103,6 @@ Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov - Stefan Beyer William Leslie Paweł Piotr Przeradowski marky1991 @@ -119,6 +120,7 @@ Wanja Saatkamp Mike Blume Gerald Klix + Julian Berman Oscar Nierstrasz Rami Chowdhury Stefan H. Muller @@ -141,6 +143,7 @@ Anton Gulenko Sergey Matyunin Andrew Chambers + Łukasz Langa Nicolas Chauvat Andrew Durdin Ben Young @@ -263,7 +266,6 @@ Bobby Impollonia Roberto De Ioris Jeong YunWon - andrewjlawrence Christopher Armstrong Aaron Tubbs Vasantha Ganesh K @@ -295,7 +297,6 @@ Ben Darnell Juan Francisco Cantero Hurtado Godefroid Chappelle - Julian Berman Stephan Busemann Dan Colish timo diff --git a/pypy/doc/release-v7.1.0.rst b/pypy/doc/release-v7.1.0.rst --- a/pypy/doc/release-v7.1.0.rst +++ b/pypy/doc/release-v7.1.0.rst @@ -14,6 +14,13 @@ The interpreters are based on much the same codebase, thus the double release. +This release, coming fast on the heels of 7.0 in February, finally merges the +internal refactoring of unicode representation as UTF-8. Removing the +conversions from strings to unicode internally lead to a nice speed bump. + +We also improved the ability to use the buffer protocol with ctype structures +and arrays. + Until we can work with downstream providers to distribute builds with PyPy, we have made packages for some common packages `available as wheels`_. diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py --- a/pypy/doc/tool/makecontributor.py +++ b/pypy/doc/tool/makecontributor.py @@ -1,4 +1,5 @@ # NOTE: run this script with LANG=en_US.UTF-8 +# works with pip install mercurial==3.0 import py import sys @@ -89,6 +90,7 @@ 'Laurence Tratt': ['ltratt'], 'Pieter Zieschang': ['pzieschang', 'p_zieschang at yahoo.de'], 'John Witulski': ['witulski'], + 'Andrew Lawrence': ['andrew.lawrence at siemens.com', 'andrewjlawrence'], } alias_map = {} diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -362,6 +362,8 @@ valid so we're trying to either raise or pack stuff with error handler. The key difference is that this is call_may_force """ + if errors is None: + errors = 'strict' slen = len(s) res = StringBuilder(slen) pos = 0 diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -526,7 +526,10 @@ def _call_codec(space, w_coder, w_obj, action, encoding, errors): try: - w_res = space.call_function(w_coder, w_obj, space.newtext(errors)) + if errors: + w_res = space.call_function(w_coder, w_obj, space.newtext(errors)) + else: + w_res = space.call_function(w_coder, w_obj) except OperationError as operr: raise _wrap_codec_error(space, operr, action, encoding) if (not space.isinstance_w(w_res, space.w_tuple) or space.len_w(w_res) != 2): @@ -634,15 +637,11 @@ return codec_info def encode_text(space, w_obj, encoding, errors): - if errors is None: - errors = 'strict' w_encoder = space.getitem( lookup_text_codec(space, "codecs.encode()", encoding), space.newint(0)) return _call_codec(space, w_encoder, w_obj, "encoding", encoding, errors) def decode_text(space, w_obj, encoding, errors): - if errors is None: - errors = 'strict' w_decoder = space.getitem( lookup_text_codec(space, "codecs.decode()", encoding), space.newint(1)) return _call_codec(space, w_decoder, w_obj, "decoding", encoding, errors) diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -696,6 +696,20 @@ exc = raises(RuntimeError, u"hello".encode, "test.failingenc") assert exc.value == to_raise + def test_one_arg_encoder(self): + import _codecs + def search_function(encoding): + def encode_one(u): + return (b'foo', len(u)) + def decode_one(u): + return (u'foo', len(u)) + if encoding == 'onearg': + return (encode_one, decode_one, None, None) + return None + _codecs.register(search_function) + assert u"hello".encode("onearg") == b'foo' + assert b"hello".decode("onearg") == 'foo' + def test_cpytest_decode(self): import codecs assert codecs.decode(b'\xe4\xf6\xfc', 'latin-1') == '\xe4\xf6\xfc' diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py --- a/pypy/objspace/std/stringmethods.py +++ b/pypy/objspace/std/stringmethods.py @@ -193,8 +193,6 @@ from pypy.objspace.std.unicodeobject import ( get_encoding_and_errors, decode_object) encoding, errors = get_encoding_and_errors(space, w_encoding, w_errors) - if errors is None: - errors = 'strict' if encoding is None: encoding = 'utf8' if encoding == 'utf8' or encoding == 'utf-8': diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -1209,7 +1209,7 @@ def encode_object(space, w_object, encoding, errors): - from pypy.module._codecs.interp_codecs import encode_text, CodecState + from pypy.module._codecs.interp_codecs import encode_text if errors is None or errors == 'strict': utf8 = space.utf8_w(w_object) if encoding is None or encoding == 'utf-8': @@ -1242,10 +1242,9 @@ return w_retval -def decode_object(space, w_obj, encoding, errors='strict'): - assert errors is not None +def decode_object(space, w_obj, encoding, errors=None): assert encoding is not None - if errors == 'strict': + if errors == 'strict' or errors is None: if encoding == 'ascii': s = space.charbuf_w(w_obj) unicodehelper.check_ascii_or_raise(space, s) @@ -1266,8 +1265,6 @@ def unicode_from_encoded_object(space, w_obj, encoding, errors): - if errors is None: - errors = 'strict' if encoding is None: encoding = getdefaultencoding(space) w_retval = decode_object(space, w_obj, encoding, errors) From pypy.commits at gmail.com Mon Mar 11 13:42:40 2019 From: pypy.commits at gmail.com (mattip) Date: Mon, 11 Mar 2019 10:42:40 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy2.7-v7.x: merge default into release Message-ID: <5c869e10.1c69fb81.5a187.46be@mx.google.com> Author: Matti Picus Branch: release-pypy2.7-v7.x Changeset: r96286:88550581f18c Date: 2019-03-11 19:41 +0200 http://bitbucket.org/pypy/pypy/changeset/88550581f18c/ Log: merge default into release diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -123,7 +123,9 @@ Wenzhu Man Konstantin Lopuhin John Witulski + Stefan Beyer Jeremy Thurgood + Andrew Lawrence Greg Price Ivan Sichmann Freitas Dario Bertini @@ -134,7 +136,6 @@ Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov - Stefan Beyer William Leslie Paweł Piotr Przeradowski marky1991 @@ -152,6 +153,7 @@ Wanja Saatkamp Mike Blume Gerald Klix + Julian Berman Oscar Nierstrasz Rami Chowdhury Stefan H. Muller @@ -174,6 +176,7 @@ Anton Gulenko Sergey Matyunin Andrew Chambers + Łukasz Langa Nicolas Chauvat Andrew Durdin Ben Young @@ -296,7 +299,6 @@ Bobby Impollonia Roberto De Ioris Jeong YunWon - andrewjlawrence Christopher Armstrong Aaron Tubbs Vasantha Ganesh K @@ -328,7 +330,6 @@ Ben Darnell Juan Francisco Cantero Hurtado Godefroid Chappelle - Julian Berman Stephan Busemann Dan Colish timo diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -90,7 +90,9 @@ Wenzhu Man Konstantin Lopuhin John Witulski + Stefan Beyer Jeremy Thurgood + Andrew Lawrence Greg Price Ivan Sichmann Freitas Dario Bertini @@ -101,7 +103,6 @@ Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov - Stefan Beyer William Leslie Paweł Piotr Przeradowski marky1991 @@ -119,6 +120,7 @@ Wanja Saatkamp Mike Blume Gerald Klix + Julian Berman Oscar Nierstrasz Rami Chowdhury Stefan H. Muller @@ -141,6 +143,7 @@ Anton Gulenko Sergey Matyunin Andrew Chambers + Łukasz Langa Nicolas Chauvat Andrew Durdin Ben Young @@ -263,7 +266,6 @@ Bobby Impollonia Roberto De Ioris Jeong YunWon - andrewjlawrence Christopher Armstrong Aaron Tubbs Vasantha Ganesh K @@ -295,7 +297,6 @@ Ben Darnell Juan Francisco Cantero Hurtado Godefroid Chappelle - Julian Berman Stephan Busemann Dan Colish timo diff --git a/pypy/doc/release-v7.1.0.rst b/pypy/doc/release-v7.1.0.rst --- a/pypy/doc/release-v7.1.0.rst +++ b/pypy/doc/release-v7.1.0.rst @@ -14,6 +14,13 @@ The interpreters are based on much the same codebase, thus the double release. +This release, coming fast on the heels of 7.0 in February, finally merges the +internal refactoring of unicode representation as UTF-8. Removing the +conversions from strings to unicode internally lead to a nice speed bump. + +We also improved the ability to use the buffer protocol with ctype structures +and arrays. + Until we can work with downstream providers to distribute builds with PyPy, we have made packages for some common packages `available as wheels`_. diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py --- a/pypy/doc/tool/makecontributor.py +++ b/pypy/doc/tool/makecontributor.py @@ -1,4 +1,5 @@ # NOTE: run this script with LANG=en_US.UTF-8 +# works with pip install mercurial==3.0 import py import sys @@ -89,6 +90,7 @@ 'Laurence Tratt': ['ltratt'], 'Pieter Zieschang': ['pzieschang', 'p_zieschang at yahoo.de'], 'John Witulski': ['witulski'], + 'Andrew Lawrence': ['andrew.lawrence at siemens.com', 'andrewjlawrence'], } alias_map = {} diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -457,6 +457,20 @@ raises(TypeError, b"hello".decode, "test.mytestenc") raises(TypeError, u"hello".encode, "test.mytestenc") + def test_one_arg_encoder(self): + import _codecs + def search_function(encoding): + def encode_one(u): + return (b'foo', len(u)) + def decode_one(u): + return (u'foo', len(u)) + if encoding == 'onearg': + return (encode_one, decode_one, None, None) + return None + _codecs.register(search_function) + assert u"hello".encode("onearg") == 'foo' + assert b"hello".decode("onearg") == 'foo' + def test_cpytest_decode(self): import codecs assert codecs.decode(b'\xe4\xf6\xfc', 'latin-1') == u'\xe4\xf6\xfc' @@ -519,7 +533,7 @@ import _codecs, array assert _codecs.readbuffer_encode(array.array('c', 'spam')) == ('spam', 4) exc = raises(TypeError, _codecs.charbuffer_encode, array.array('c', 'spam')) - assert str(exc.value) == "must be string or read-only character buffer, not array.array" + assert "must be string or read-only character buffer, not array.array" in str(exc.value) assert _codecs.readbuffer_encode(u"test") == ('test', 4) assert _codecs.charbuffer_encode(u"test") == ('test', 4) diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -1073,10 +1073,6 @@ def encode_object(space, w_object, encoding, errors): w_encoder = None - if encoding is None: - # Get the encoder functions as a wrapped object. - # This lookup is cached. - w_encoder = space.sys.get_w_default_encoder() if errors is None or errors == 'strict': if ((encoding is None and space.sys.defaultencoding == 'ascii') or encoding == 'ascii'): @@ -1101,14 +1097,18 @@ if rutf8.has_surrogates(utf8): utf8 = rutf8.reencode_utf8_with_surrogates(utf8) return space.newbytes(utf8) + if encoding is None: + # Get the encoder functions as a wrapped object. + # This lookup is cached. + w_encoder = space.sys.get_w_default_encoder() if w_encoder is None: from pypy.module._codecs.interp_codecs import lookup_codec w_encoder = space.getitem(lookup_codec(space, encoding), space.newint(0)) if errors is None: - w_errors = space.newtext('strict') + w_restuple = space.call_function(w_encoder, w_object) else: w_errors = space.newtext(errors) - w_restuple = space.call_function(w_encoder, w_object, w_errors) + w_restuple = space.call_function(w_encoder, w_object, w_errors) w_retval = space.getitem(w_restuple, space.newint(0)) if not space.isinstance_w(w_retval, space.w_bytes): raise oefmt(space.w_TypeError, @@ -1118,9 +1118,9 @@ def decode_object(space, w_obj, encoding, errors): - if encoding is None: - encoding = getdefaultencoding(space) if errors is None or errors == 'strict': + if encoding is None: + encoding = getdefaultencoding(space) if encoding == 'ascii': s = space.charbuf_w(w_obj) unicodehelper.check_ascii_or_raise(space, s) @@ -1133,14 +1133,19 @@ s = space.charbuf_w(w_obj) lgt = unicodehelper.check_utf8_or_raise(space, s) return space.newutf8(s, lgt) - w_codecs = space.getbuiltinmodule("_codecs") - w_decode = space.getattr(w_codecs, space.newtext("decode")) + w_decoder = None + if encoding is None: + # Get the decoder functions as a wrapped object. + # This lookup is cached. + w_decoder = space.sys.get_w_default_decoder() + if w_decoder is None: + from pypy.module._codecs.interp_codecs import lookup_codec + w_decoder = space.getitem(lookup_codec(space, encoding), space.newint(1)) if errors is None: - w_retval = space.call_function(w_decode, w_obj, space.newtext(encoding)) + w_retval = space.call_function(w_decoder, w_obj) else: - w_retval = space.call_function(w_decode, w_obj, space.newtext(encoding), - space.newtext(errors)) - return w_retval + w_retval = space.call_function(w_decoder, w_obj, space.newtext(errors)) + return space.getitem(w_retval, space.newint(0)) def unicode_from_encoded_object(space, w_obj, encoding, errors): From pypy.commits at gmail.com Tue Mar 12 04:00:28 2019 From: pypy.commits at gmail.com (andrewjlawrence) Date: Tue, 12 Mar 2019 01:00:28 -0700 (PDT) Subject: [pypy-commit] pypy winoverlapped: Further implementation of _overlapped Message-ID: <5c87671c.1c69fb81.c1c0c.fca6@mx.google.com> Author: andrewjlawrence Branch: winoverlapped Changeset: r96288:c4d3d27effa0 Date: 2019-03-02 21:39 +0000 http://bitbucket.org/pypy/pypy/changeset/c4d3d27effa0/ Log: Further implementation of _overlapped diff --git a/lib_pypy/_overlapped.py b/lib_pypy/_overlapped.py --- a/lib_pypy/_overlapped.py +++ b/lib_pypy/_overlapped.py @@ -21,6 +21,16 @@ from _winapi import INVALID_HANDLE_VALUE, _MAX_PATH , _Z import _winapi +# +# Error Codes +# +ERROR_PIPE_BUSY = 231 + +# +# Status Codes +# +STATUS_PENDING = 0x00000103 + DisconnectEx = _ffi.NULL def _int2intptr(int2cast): @@ -62,23 +72,27 @@ self.type = OverlappedType.TYPE_NONE self.overlapped[0].hEvent = \ _kernel32.CreateEventW(NULL, True, False, NULL) + if self.overlapped[0].hEvent == _ffi.NULL: + raise _winapi._WinError() self.address = _ffi.addressof(self.overlapped[0]) def __del__(self): - bytes = _ffi.new("DWORD") - olderr = _winapi.GetLastError() - if not HasOverlappedIoCompleted(self.overlapped) and self.type != TYPE_NOT_STARTED: + bytes = _ffi.new("DWORD[1]",[0]) + olderr = _kernel32.GetLastError() + hascompletedio = HasOverlappedIoCompleted(self.overlapped[0]) + if not hascompletedio and self.type != TYPE_NOT_STARTED: wait = _kernel32.CancelIoEx(self.handle, self.overlapped) ret = self.GetOverlappedResult(wait) err = _winapi.ERROR_SUCCESS if not ret: - err = _winapi.GetLastError() + err = _kernel32.GetLastError() if err != _winapi.ERROR_SUCCESS and \ err != _winapi.ERROR_NOT_FOUND and \ err != _winapi.ERROR_OPERATION_ABORTED: raise _winapi._WinError() - if self.overlapped.hEvent: - _winapi.CloseHandle(self.overlapped.hEvent) + + if self.overlapped[0].hEvent: + _winapi.CloseHandle(self.overlapped[0].hEvent) @property def event(self): @@ -86,8 +100,6 @@ def GetOverlappedResult(self, wait): transferred = _ffi.new('DWORD[1]', [0]) - - res = _kernel32.GetOverlappedResult(self.handle, self.overlapped, transferred, wait != 0) if res: err = _winapi.ERROR_SUCCESS @@ -157,46 +169,73 @@ def getresult(self, wait=False): return self.GetOverlappedResult(wait) + + def ConnectNamedPipe(self, handle, overlapped=False): + if overlapped: + ov = Overlapped(handle) + else: + ov = Overlapped(None) + self.type = OverlappedType.TYPE_CONNECT_NAMED_PIPE + + success = _kernel32.ConnectNamedPipe(handle, ov.overlapped) + + if overlapped and err == _winapi.ERROR_IO_PENDING: + ov.pending = 1 + + err = _kernel32.GetLastError() + if err == _winapi.ERROR_IO_PENDING | _winapi.ERROR_SUCCESS: + return False + elif err == _winapi.ERROR_PIPE_CONNECTED: + mark_as_completed(self.overlapped) + return True + else: + raise _winapi._WinError() + + def ReadFile(self, handle, size): + self.type = OverlappedType.TYPE_READ + self.handle = handle + self.allocated_buffer = _ffi.new("CHAR[]", max(1,size)) + return self.do_ReadFile(handle, self.allocated_buffer, size) + + def do_ReadFile(self, handle, buf, size): + nread = _ffi.new('DWORD[1]', [0]) + ret = _kernel32.ReadFile(handle, buf, size, nread, self.overlapped) + if ret: + err = _winapi.ERROR_SUCCESS + else: + err = _kernel32.GetLastError() + + if err == _winapi.ERROR_BROKEN_PIPE: + mark_as_completed(self.overlapped) + raise _winapi._WinError() + elif err in [_winapi.ERROR_SUCCESS, _winapi.ERROR_MORE_DATA, _winapi.ERROR_IO_PENDING]: + return None + else: + self.type = OverlappedType.TYPE_NOT_STARTED + raise _winapi._WinError() + def mark_as_completed(overlapped): - overlapped.overlapped.Internal = _ffi.NULL - if overlapped.overlapped.hEvent != _ffi.NULL: - SetEvent(overlapped.overlapped.hEvent) + overlapped[0].Internal = 0 + if overlapped[0].hEvent != _ffi.NULL: + SetEvent(overlapped[0].hEvent) + +def SetEvent(handle): + ret = _kernel32.SetEvent(handle) + if not ret: + raise _winapi._WinError() def CreateEvent(eventattributes, manualreset, initialstate, name): event = _kernel32.CreateEventW(NULL, manualreset, initialstate, _Z(name)) if not event: raise _winapi._WinError() return event - -def ConnectNamedPipe(handle, overlapped=False): - if overlapped: - ov = Overlapped(handle) - else: - ov = Overlapped(None) - - success = _kernel32.ConnectNamedPipe(handle, ov.overlapped) - if overlapped: - # Overlapped ConnectNamedPipe never returns a success code - assert success == 0 - err = _kernel32.GetLastError() - if err == _winapi.ERROR_IO_PENDING: - ov.pending = 1 - elif err == _winapi.ERROR_PIPE_CONNECTED: - _kernel32.SetEvent(ov.overlapped[0].hEvent) - else: - del ov - raise _winapi._WinError() - return ov - elif not success: - raise _winapi._WinError() def CreateIoCompletionPort(handle, existingcompletionport, completionkey, numberofconcurrentthreads): completionkey = _int2intptr(completionkey) existingcompletionport = _int2handle(existingcompletionport) numberofconcurrentthreads = _int2dword(numberofconcurrentthreads) handle = _int2handle(handle) - result = _kernel32.CreateIoCompletionPort(handle, existingcompletionport, completionkey, @@ -213,6 +252,7 @@ if completionport is None: raise _winapi._WinError() overlapped = _ffi.new("OVERLAPPED **") + overlapped[0] = _ffi.NULL result = _kernel32.GetQueuedCompletionStatus(completionport, numberofbytes, completionkey, @@ -222,6 +262,7 @@ err = _winapi.ERROR_SUCCESS else: err = _kernel32.GetLastError() + if overlapped[0] == _ffi.NULL: if err == _winapi.WAIT_TIMEOUT: return None @@ -251,15 +292,22 @@ return newwaitobject def ConnectPipe(address): + err = _winapi.ERROR_PIPE_BUSY + handle = _kernel32.CreateFileW(address, - _winapi.GENERIC_READ | _winapi.GENERIC_WRITE, - 0, - _ffi.NULL, - _winapi.OPEN_EXISTING, - _winapi.FILE_FLAG_OVERLAPPED, - _ffi.NULL) + _winapi.GENERIC_READ | _winapi.GENERIC_WRITE, + 0, + _ffi.NULL, + _winapi.OPEN_EXISTING, + _winapi.FILE_FLAG_OVERLAPPED, + _ffi.NULL) + err = _kernel32.GetLastError() + + if handle == INVALID_HANDLE_VALUE: + raise _winapi._WinError() + return handle - + # In CPython this function converts a windows error into a python object # Not sure what we should do here. @@ -270,7 +318,4 @@ return (overlapped.Internal != STATUS_PENDING) -# -# Error Codes -# -ERROR_PIPE_BUSY = 231 \ No newline at end of file + diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py --- a/lib_pypy/_pypy_winbase_build.py +++ b/lib_pypy/_pypy_winbase_build.py @@ -100,11 +100,12 @@ DWORD, DWORD, HANDLE); HANDLE WINAPI CreateFileW(LPCWSTR, DWORD, DWORD, LPSECURITY_ATTRIBUTES, DWORD, DWORD, HANDLE); +BOOL ReadFile(HANDLE, LPVOID, DWORD, LPDWORD, LPOVERLAPPED); BOOL WINAPI SetNamedPipeHandleState(HANDLE, LPDWORD, LPDWORD, LPDWORD); BOOL WINAPI ConnectNamedPipe(HANDLE, LPOVERLAPPED); HANDLE WINAPI CreateEventA(LPSECURITY_ATTRIBUTES, BOOL, BOOL, LPCSTR); HANDLE WINAPI CreateEventW(LPSECURITY_ATTRIBUTES, BOOL, BOOL, LPCWSTR); -VOID WINAPI SetEvent(HANDLE); +BOOL WINAPI SetEvent(HANDLE); BOOL WINAPI CancelIo(HANDLE); BOOL WINAPI CancelIoEx(HANDLE, LPOVERLAPPED); BOOL WINAPI CloseHandle(HANDLE); diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py --- a/lib_pypy/_pypy_winbase_cffi.py +++ b/lib_pypy/_pypy_winbase_cffi.py @@ -3,8 +3,8 @@ ffi = _cffi_backend.FFI('_pypy_winbase_cffi', _version = 0x2601, - _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\xE9\x03\x00\x00\x13\x11\x00\x00\xEF\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\xE7\x03\x00\x00\xE3\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\xDE\x03\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\xE2\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x31\x11\x00\x00\x18\x03\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x31\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\xE8\x03\x00\x00\x0A\x01\x00\x00\x36\x11\x00\x00\x36\x11\x00\x00\x31\x11\x00\x00\xD6\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x36\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x36\x11\x00\x00\x36\x03\x00\x00\x31\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x36\x11\x00\x00\x36\x11\x00\x00\x36\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x31\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x8F\x03\x00\x00\x6D\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x6D\x11\x00\x00\x6D\x11\x00\x00\x1B\x11\x00\x00\x1C\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x4A\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x6D\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x8A\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x8A\x0D\x00\x00\x00\x0F\x00\x00\x8A\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\xE6\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xE9\x03\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x92\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x8F\x03\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x95\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x92\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x95\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x92\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x6D\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x92\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x9B\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x92\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\xEF\x0D\x00\x00\x31\x11\x00\x00\x00\x0F\x00\x00\xEF\x0D\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x00\x0F\x00\x00\xEF\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\xEF\x0D\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x04\x09\x00\x00\x02\x09\x00\x00\xE5\x03\x00\x00\x05\x09\x00\x00\x06\x09\x00\x00\x03\x09\x00\x00\x07\x09\x00\x00\x02\x01\x00\x00\x39\x03\x00\x00\x01\x09\x00\x00\x00\x09\x00\x00\xEE\x03\x00\x00\x04\x01\x00\x00\x00\x01', - _globals = (b'\x00\x00\x2C\x23CancelIo',0,b'\x00\x00\x2F\x23CancelIoEx',0,b'\x00\x00\x2C\x23CloseHandle',0,b'\x00\x00\x2F\x23ConnectNamedPipe',0,b'\x00\x00\x91\x23CreateEventA',0,b'\x00\x00\x97\x23CreateEventW',0,b'\x00\x00\x9D\x23CreateFileA',0,b'\x00\x00\xCA\x23CreateFileW',0,b'\x00\x00\xB8\x23CreateIoCompletionPort',0,b'\x00\x00\xA6\x23CreateNamedPipeA',0,b'\x00\x00\xC0\x23CreateNamedPipeW',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x6C\x23CreateProcessW',0,b'\x00\x00\x63\x23DuplicateHandle',0,b'\x00\x00\xBE\x23GetCurrentProcess',0,b'\x00\x00\x4C\x23GetExitCodeProcess',0,b'\x00\x00\x87\x23GetLastError',0,b'\x00\x00\x82\x23GetModuleFileNameW',0,b'\x00\x00\x33\x23GetOverlappedResult',0,b'\x00\x00\x50\x23GetQueuedCompletionStatus',0,b'\x00\x00\xB5\x23GetStdHandle',0,b'\x00\x00\x87\x23GetVersion',0,b'\x00\x00\xD3\x23HasOverlappedIoCompleted',0,b'\x00\x00\x5D\x23PostQueuedCompletionStatus',0,b'\x00\x00\x24\x23RegisterWaitForSingleObject',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\x7B\x23SetErrorMode',0,b'\x00\x00\xDB\x23SetEvent',0,b'\x00\x00\x57\x23SetNamedPipeHandleState',0,b'\x00\x00\x48\x23TerminateProcess',0,b'\x00\x00\x3F\x23WSARecv',0,b'\xFF\xFF\xFF\x1FWT_EXECUTEINWAITTHREAD',4,b'\xFF\xFF\xFF\x1FWT_EXECUTEONLYONCE',8,b'\x00\x00\x7E\x23WaitForSingleObject',0,b'\x00\x00\x78\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x8C\x23_getwch',0,b'\x00\x00\x8C\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x8E\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x89\x23_ungetwch',0,b'\x00\x00\xB0\x23socket',0), - _struct_unions = ((b'\x00\x00\x00\xEC\x00\x00\x00\x03$1',b'\x00\x00\xEB\x11DUMMYSTRUCTNAME',b'\x00\x00\x15\x11Pointer'),(b'\x00\x00\x00\xEB\x00\x00\x00\x02$2',b'\x00\x00\x18\x11Offset',b'\x00\x00\x18\x11OffsetHigh'),(b'\x00\x00\x00\xE3\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\xE7\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x8A\x11wShowWindow',b'\x00\x00\x8A\x11cbReserved2',b'\x00\x00\xED\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError'),(b'\x00\x00\x00\xE2\x00\x00\x00\x02_OVERLAPPED',b'\x00\x00\x18\x11Internal',b'\x00\x00\x18\x11InternalHigh',b'\x00\x00\xEC\x11DUMMYUNIONNAME',b'\x00\x00\x15\x11hEvent'),(b'\x00\x00\x00\xE5\x00\x00\x00\x02_PostCallbackData',b'\x00\x00\x15\x11hCompletionPort',b'\x00\x00\x31\x11Overlapped'),(b'\x00\x00\x00\xE6\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x18\x11nLength',b'\x00\x00\x15\x11lpSecurityDescriptor',b'\x00\x00\x01\x11bInheritHandle'),(b'\x00\x00\x00\xE8\x00\x00\x00\x02_WSABUF',b'\x00\x00\x18\x11len',b'\x00\x00\x13\x11buf')), - _typenames = (b'\x00\x00\x00\xEALPFN_DISCONNECTEX',b'\x00\x00\x00\x31LPOVERLAPPED',b'\x00\x00\x00\x46LPOVERLAPPED_COMPLETION_ROUTINE',b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\xE4LPPostCallbackData',b'\x00\x00\x00\x92LPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\x41LPWSABUF',b'\x00\x00\x00\xE2OVERLAPPED',b'\x00\x00\x00\xE3PROCESS_INFORMATION',b'\x00\x00\x00\x92PSECURITY_ATTRIBUTES',b'\x00\x00\x00\xE5PostCallbackData',b'\x00\x00\x00\xE6SECURITY_ATTRIBUTES',b'\x00\x00\x00\x15SOCKET',b'\x00\x00\x00\xE7STARTUPINFO',b'\x00\x00\x00\x27WAITORTIMERCALLBACK',b'\x00\x00\x00\xE8WSABUF',b'\x00\x00\x00\x8Awint_t'), + _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\xF0\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x13\x03\x00\x00\xE3\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\xEA\x03\x00\x00\x1A\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x1A\x11\x00\x00\x1A\x11\x00\x00\xE8\x03\x00\x00\xE4\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x03\x00\x00\x26\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x26\x11\x00\x00\x11\x11\x00\x00\xDF\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x14\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\xE9\x03\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\xDA\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x14\x03\x00\x00\x15\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x26\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x96\x03\x00\x00\x74\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x74\x11\x00\x00\x74\x11\x00\x00\x22\x11\x00\x00\x23\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x51\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x11\x11\x00\x00\x74\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x02\x0F\x00\x00\x91\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x91\x0D\x00\x00\x00\x0F\x00\x00\x91\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x00\xE7\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xEA\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x99\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x96\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x9C\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x99\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x9C\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x99\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x74\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x99\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xA2\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x99\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\xF0\x0D\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x00\x0F\x00\x00\xF0\x0D\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x04\x09\x00\x00\x02\x09\x00\x00\xE6\x03\x00\x00\x05\x09\x00\x00\x06\x09\x00\x00\x03\x09\x00\x00\x07\x09\x00\x00\x02\x01\x00\x00\x40\x03\x00\x00\x01\x09\x00\x00\x00\x09\x00\x00\xEF\x03\x00\x00\x04\x01\x00\x00\x00\x01', + _globals = (b'\x00\x00\x33\x23CancelIo',0,b'\x00\x00\x36\x23CancelIoEx',0,b'\x00\x00\x33\x23CloseHandle',0,b'\x00\x00\x36\x23ConnectNamedPipe',0,b'\x00\x00\x98\x23CreateEventA',0,b'\x00\x00\x9E\x23CreateEventW',0,b'\x00\x00\xA4\x23CreateFileA',0,b'\x00\x00\xD1\x23CreateFileW',0,b'\x00\x00\xBF\x23CreateIoCompletionPort',0,b'\x00\x00\xAD\x23CreateNamedPipeA',0,b'\x00\x00\xC7\x23CreateNamedPipeW',0,b'\x00\x00\x25\x23CreatePipe',0,b'\x00\x00\x19\x23CreateProcessA',0,b'\x00\x00\x73\x23CreateProcessW',0,b'\x00\x00\x6A\x23DuplicateHandle',0,b'\x00\x00\xC5\x23GetCurrentProcess',0,b'\x00\x00\x53\x23GetExitCodeProcess',0,b'\x00\x00\x8E\x23GetLastError',0,b'\x00\x00\x89\x23GetModuleFileNameW',0,b'\x00\x00\x3A\x23GetOverlappedResult',0,b'\x00\x00\x57\x23GetQueuedCompletionStatus',0,b'\x00\x00\xBC\x23GetStdHandle',0,b'\x00\x00\x8E\x23GetVersion',0,b'\x00\x00\x64\x23PostQueuedCompletionStatus',0,b'\x00\x00\x10\x23ReadFile',0,b'\x00\x00\x2B\x23RegisterWaitForSingleObject',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\x82\x23SetErrorMode',0,b'\x00\x00\x33\x23SetEvent',0,b'\x00\x00\x5E\x23SetNamedPipeHandleState',0,b'\x00\x00\x4F\x23TerminateProcess',0,b'\x00\x00\x46\x23WSARecv',0,b'\xFF\xFF\xFF\x1FWT_EXECUTEINWAITTHREAD',4,b'\xFF\xFF\xFF\x1FWT_EXECUTEONLYONCE',8,b'\x00\x00\x85\x23WaitForSingleObject',0,b'\x00\x00\x7F\x23_get_osfhandle',0,b'\x00\x00\x17\x23_getch',0,b'\x00\x00\x17\x23_getche',0,b'\x00\x00\x93\x23_getwch',0,b'\x00\x00\x93\x23_getwche',0,b'\x00\x00\x17\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x95\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x90\x23_ungetwch',0,b'\x00\x00\xB7\x23socket',0), + _struct_unions = ((b'\x00\x00\x00\xED\x00\x00\x00\x03$1',b'\x00\x00\xEC\x11DUMMYSTRUCTNAME',b'\x00\x00\x11\x11Pointer'),(b'\x00\x00\x00\xEC\x00\x00\x00\x02$2',b'\x00\x00\x13\x11Offset',b'\x00\x00\x13\x11OffsetHigh'),(b'\x00\x00\x00\xE4\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x11\x11hProcess',b'\x00\x00\x11\x11hThread',b'\x00\x00\x13\x11dwProcessId',b'\x00\x00\x13\x11dwThreadId'),(b'\x00\x00\x00\xE8\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x13\x11cb',b'\x00\x00\x1A\x11lpReserved',b'\x00\x00\x1A\x11lpDesktop',b'\x00\x00\x1A\x11lpTitle',b'\x00\x00\x13\x11dwX',b'\x00\x00\x13\x11dwY',b'\x00\x00\x13\x11dwXSize',b'\x00\x00\x13\x11dwYSize',b'\x00\x00\x13\x11dwXCountChars',b'\x00\x00\x13\x11dwYCountChars',b'\x00\x00\x13\x11dwFillAttribute',b'\x00\x00\x13\x11dwFlags',b'\x00\x00\x91\x11wShowWindow',b'\x00\x00\x91\x11cbReserved2',b'\x00\x00\xEE\x11lpReserved2',b'\x00\x00\x11\x11hStdInput',b'\x00\x00\x11\x11hStdOutput',b'\x00\x00\x11\x11hStdError'),(b'\x00\x00\x00\xE3\x00\x00\x00\x02_OVERLAPPED',b'\x00\x00\x13\x11Internal',b'\x00\x00\x13\x11InternalHigh',b'\x00\x00\xED\x11DUMMYUNIONNAME',b'\x00\x00\x11\x11hEvent'),(b'\x00\x00\x00\xE6\x00\x00\x00\x02_PostCallbackData',b'\x00\x00\x11\x11hCompletionPort',b'\x00\x00\x15\x11Overlapped'),(b'\x00\x00\x00\xE7\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x13\x11nLength',b'\x00\x00\x11\x11lpSecurityDescriptor',b'\x00\x00\x01\x11bInheritHandle'),(b'\x00\x00\x00\xE9\x00\x00\x00\x02_WSABUF',b'\x00\x00\x13\x11len',b'\x00\x00\x1A\x11buf')), + _typenames = (b'\x00\x00\x00\xEBLPFN_DISCONNECTEX',b'\x00\x00\x00\x15LPOVERLAPPED',b'\x00\x00\x00\x4DLPOVERLAPPED_COMPLETION_ROUTINE',b'\x00\x00\x00\x23LPPROCESS_INFORMATION',b'\x00\x00\x00\xE5LPPostCallbackData',b'\x00\x00\x00\x99LPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x22LPSTARTUPINFO',b'\x00\x00\x00\x48LPWSABUF',b'\x00\x00\x00\xE3OVERLAPPED',b'\x00\x00\x00\xE4PROCESS_INFORMATION',b'\x00\x00\x00\x99PSECURITY_ATTRIBUTES',b'\x00\x00\x00\xE6PostCallbackData',b'\x00\x00\x00\xE7SECURITY_ATTRIBUTES',b'\x00\x00\x00\x11SOCKET',b'\x00\x00\x00\xE8STARTUPINFO',b'\x00\x00\x00\x2EWAITORTIMERCALLBACK',b'\x00\x00\x00\xE9WSABUF',b'\x00\x00\x00\x91wint_t'), ) diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py --- a/lib_pypy/_winapi.py +++ b/lib_pypy/_winapi.py @@ -33,7 +33,7 @@ def CreatePipe(attributes, size): handles = _ffi.new("HANDLE[2]") - + res = _kernel32.CreatePipe(handles, handles + 1, NULL, size) if not res: @@ -204,12 +204,14 @@ ERROR_SUCCESS = 0 ERROR_NETNAME_DELETED = 64 ERROR_BROKEN_PIPE = 109 +ERROR_PIPE_BUSY = 231 ERROR_MORE_DATA = 234 ERROR_PIPE_CONNECTED = 535 ERROR_OPERATION_ABORTED = 995 ERROR_IO_INCOMPLETE = 996 ERROR_IO_PENDING = 997 + PIPE_ACCESS_INBOUND = 0x00000001 PIPE_ACCESS_OUTBOUND = 0x00000002 PIPE_ACCESS_DUPLEX = 0x00000003 From pypy.commits at gmail.com Tue Mar 12 04:03:08 2019 From: pypy.commits at gmail.com (mattip) Date: Tue, 12 Mar 2019 01:03:08 -0700 (PDT) Subject: [pypy-commit] pypy default: reduce code duplication Message-ID: <5c8767bc.1c69fb81.2899a.3ea7@mx.google.com> Author: Matti Picus Branch: Changeset: r96292:0a08f62b2c9f Date: 2019-03-12 10:02 +0200 http://bitbucket.org/pypy/pypy/changeset/0a08f62b2c9f/ Log: reduce code duplication diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -306,8 +306,8 @@ return w_err_handler - at unwrap_spec(errors='text') -def encode(space, w_obj, w_encoding=None, errors='strict'): + at unwrap_spec(encoding='text_or_none', errors='text_or_none') +def encode(space, w_obj, encoding=None, errors=None): """encode(obj, [encoding[,errors]]) -> object Encodes obj using the codec registered for encoding. encoding defaults @@ -317,13 +317,19 @@ 'xmlcharrefreplace' as well as any other name registered with codecs.register_error that can handle ValueErrors. """ - if w_encoding is None: + if encoding is None: encoding = space.sys.defaultencoding + w_encoder = space.getitem(lookup_codec(space, encoding), space.newint(0)) + if errors: + w_res = space.call_function(w_encoder, w_obj, space.newtext(errors)) else: - encoding = space.text_w(w_encoding) - w_encoder = space.getitem(lookup_codec(space, encoding), space.newint(0)) - w_res = space.call_function(w_encoder, w_obj, space.newtext(errors)) - return space.getitem(w_res, space.newint(0)) + w_res = space.call_function(w_encoder, w_obj) + w_retval = space.getitem(w_res, space.newint(0)) + if not space.isinstance_w(w_retval, space.w_bytes): + raise oefmt(space.w_TypeError, + "encoder did not return an string object (type '%T')", + w_retval) + return w_retval @unwrap_spec(errors='text_or_none') def readbuffer_encode(space, w_data, errors='strict'): @@ -335,8 +341,8 @@ s = space.getarg_w('t#', w_data) return space.newtuple([space.newbytes(s), space.newint(len(s))]) - at unwrap_spec(errors='text') -def decode(space, w_obj, w_encoding=None, errors='strict'): + at unwrap_spec(encoding='text_or_none', errors='text_or_none') +def decode(space, w_obj, encoding=None, errors=None): """decode(obj, [encoding[,errors]]) -> object Decodes obj using the codec registered for encoding. encoding defaults @@ -346,19 +352,17 @@ as well as any other name registered with codecs.register_error that is able to handle ValueErrors. """ - if w_encoding is None: + if encoding is None: encoding = space.sys.defaultencoding + w_decoder = space.getitem(lookup_codec(space, encoding), space.newint(1)) + if errors: + w_res = space.call_function(w_decoder, w_obj, space.newtext(errors)) else: - encoding = space.text_w(w_encoding) - w_decoder = space.getitem(lookup_codec(space, encoding), space.newint(1)) - if space.is_true(w_decoder): - w_res = space.call_function(w_decoder, w_obj, space.newtext(errors)) - if (not space.isinstance_w(w_res, space.w_tuple) or space.len_w(w_res) != 2): - raise oefmt(space.w_TypeError, - "encoder must return a tuple (object, integer)") - return space.getitem(w_res, space.newint(0)) - else: - assert 0, "XXX, what to do here?" + w_res = space.call_function(w_decoder, w_obj) + if (not space.isinstance_w(w_res, space.w_tuple) or space.len_w(w_res) != 2): + raise oefmt(space.w_TypeError, + "encoder must return a tuple (object, integer)") + return space.getitem(w_res, space.newint(0)) @unwrap_spec(errors='text') def register_error(space, errors, w_handler): diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -468,8 +468,10 @@ return (encode_one, decode_one, None, None) return None _codecs.register(search_function) - assert u"hello".encode("onearg") == 'foo' - assert b"hello".decode("onearg") == 'foo' + assert u"hello".encode("onearg") == b'foo' + assert b"hello".decode("onearg") == u'foo' + assert _codecs.encode(u"hello", "onearg") == b'foo' + assert _codecs.decode(b"hello", "onearg") == u'foo' def test_cpytest_decode(self): import codecs diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -1071,16 +1071,17 @@ return encoding, errors -def encode_object(space, w_object, encoding, errors): - w_encoder = None +def encode_object(space, w_obj, encoding, errors): + from pypy.module._codecs.interp_codecs import encode if errors is None or errors == 'strict': + # fast path if ((encoding is None and space.sys.defaultencoding == 'ascii') or encoding == 'ascii'): - s = space.utf8_w(w_object) + s = space.utf8_w(w_obj) try: rutf8.check_ascii(s) except rutf8.CheckError as a: - if space.isinstance_w(w_object, space.w_unicode): + if space.isinstance_w(w_obj, space.w_unicode): eh = unicodehelper.encode_error_handler(space) else: # must be a bytes-like object. In order to encode it, @@ -1093,32 +1094,17 @@ return space.newbytes(s) if ((encoding is None and space.sys.defaultencoding == 'utf8') or encoding == 'utf-8' or encoding == 'utf8' or encoding == 'UTF-8'): - utf8 = space.utf8_w(w_object) + utf8 = space.utf8_w(w_obj) if rutf8.has_surrogates(utf8): utf8 = rutf8.reencode_utf8_with_surrogates(utf8) return space.newbytes(utf8) - if encoding is None: - # Get the encoder functions as a wrapped object. - # This lookup is cached. - w_encoder = space.sys.get_w_default_encoder() - if w_encoder is None: - from pypy.module._codecs.interp_codecs import lookup_codec - w_encoder = space.getitem(lookup_codec(space, encoding), space.newint(0)) - if errors is None: - w_restuple = space.call_function(w_encoder, w_object) - else: - w_errors = space.newtext(errors) - w_restuple = space.call_function(w_encoder, w_object, w_errors) - w_retval = space.getitem(w_restuple, space.newint(0)) - if not space.isinstance_w(w_retval, space.w_bytes): - raise oefmt(space.w_TypeError, - "encoder did not return an string object (type '%T')", - w_retval) - return w_retval + return encode(space, w_obj, encoding, errors) def decode_object(space, w_obj, encoding, errors): + from pypy.module._codecs.interp_codecs import lookup_codec, decode if errors is None or errors == 'strict': + # fast paths if encoding is None: encoding = getdefaultencoding(space) if encoding == 'ascii': @@ -1133,20 +1119,9 @@ s = space.charbuf_w(w_obj) lgt = unicodehelper.check_utf8_or_raise(space, s) return space.newutf8(s, lgt) - w_decoder = None if encoding is None: - # Get the decoder functions as a wrapped object. - # This lookup is cached. - w_decoder = space.sys.get_w_default_decoder() - if w_decoder is None: - from pypy.module._codecs.interp_codecs import lookup_codec - w_decoder = space.getitem(lookup_codec(space, encoding), space.newint(1)) - if errors is None: - w_retval = space.call_function(w_decoder, w_obj) - else: - w_retval = space.call_function(w_decoder, w_obj, space.newtext(errors)) - return space.getitem(w_retval, space.newint(0)) - + encoding = space.sys.defaultencoding + return decode(space, w_obj, encoding, errors) def unicode_from_encoded_object(space, w_obj, encoding, errors): # explicitly block bytearray on 2.7 From pypy.commits at gmail.com Tue Mar 12 07:44:41 2019 From: pypy.commits at gmail.com (Julian Berman) Date: Tue, 12 Mar 2019 04:44:41 -0700 (PDT) Subject: [pypy-commit] pypy zlib-make-py3-go-boom: Complain if you try to copy a flushed zlib decompress on py3. Message-ID: <5c879ba9.1c69fb81.643af.73cb@mx.google.com> Author: Julian Berman Branch: zlib-make-py3-go-boom Changeset: r96294:0bb579c51e8b Date: 2019-03-12 07:43 -0400 http://bitbucket.org/pypy/pypy/changeset/0bb579c51e8b/ Log: Complain if you try to copy a flushed zlib decompress on py3. diff --git a/pypy/module/zlib/interp_zlib.py b/pypy/module/zlib/interp_zlib.py --- a/pypy/module/zlib/interp_zlib.py +++ b/pypy/module/zlib/interp_zlib.py @@ -313,6 +313,11 @@ try: self.lock() try: + if not self.stream: + raise oefmt( + space.w_ValueError, + "Decompressor was already flushed", + ) copied = rzlib.inflateCopy(self.stream) finally: self.unlock() From pypy.commits at gmail.com Tue Mar 12 15:14:03 2019 From: pypy.commits at gmail.com (rlamy) Date: Tue, 12 Mar 2019 12:14:03 -0700 (PDT) Subject: [pypy-commit] pypy apptest-file: hg merge default Message-ID: <5c8804fb.1c69fb81.c306f.bc8a@mx.google.com> Author: Ronan Lamy Branch: apptest-file Changeset: r96297:73c5a033c44d Date: 2019-03-12 18:18 +0000 http://bitbucket.org/pypy/pypy/changeset/73c5a033c44d/ Log: hg merge default diff too long, truncating to 2000 out of 229297 lines diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -4,8 +4,10 @@ *~ .*.swp .idea +.mypy_cache .project .pydevproject +.vscode __pycache__ .cache/ diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -58,3 +58,12 @@ 3f6eaa010fce78cc7973bdc1dfdb95970f08fed2 release-pypy3.5-v5.10.1 ab0b9caf307db6592905a80b8faffd69b39005b8 release-pypy2.7-v6.0.0 fdd60ed87e941677e8ea11acf9f1819466521bf2 release-pypy3.5-v6.0.0 +9112c8071614108b1042bfef0713915107004d62 release-pypy2.7-v7.0.0 +1f86f25937b6ae6c8b25236c35228fac587678bf release-pypy3.5-v7.0.0 +dab365a465140aa79a5f3ba4db784c4af4d5c195 release-pypy3.6-v7.0.0 +9112c8071614108b1042bfef0713915107004d62 release-pypy2.7-v7.0.0 +c8805ee6d7846ca2722b106eeaa2f128c699aba3 release-pypy2.7-v7.0.0 +1f86f25937b6ae6c8b25236c35228fac587678bf release-pypy3.5-v7.0.0 +928a4f70d3de7d17449456946154c5da6e600162 release-pypy3.5-v7.0.0 +dab365a465140aa79a5f3ba4db784c4af4d5c195 release-pypy3.6-v7.0.0 +fb40f7a5524c77b80e6c468e087d621610137261 release-pypy3.6-v7.0.0 diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -123,7 +123,9 @@ Wenzhu Man Konstantin Lopuhin John Witulski + Stefan Beyer Jeremy Thurgood + Andrew Lawrence Greg Price Ivan Sichmann Freitas Dario Bertini @@ -134,7 +136,6 @@ Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov - Stefan Beyer William Leslie Paweł Piotr Przeradowski marky1991 @@ -152,6 +153,7 @@ Wanja Saatkamp Mike Blume Gerald Klix + Julian Berman Oscar Nierstrasz Rami Chowdhury Stefan H. Muller @@ -174,6 +176,7 @@ Anton Gulenko Sergey Matyunin Andrew Chambers + Łukasz Langa Nicolas Chauvat Andrew Durdin Ben Young @@ -296,7 +299,6 @@ Bobby Impollonia Roberto De Ioris Jeong YunWon - andrewjlawrence Christopher Armstrong Aaron Tubbs Vasantha Ganesh K @@ -328,7 +330,6 @@ Ben Darnell Juan Francisco Cantero Hurtado Godefroid Chappelle - Julian Berman Stephan Busemann Dan Colish timo diff --git a/TODO b/TODO new file mode 100644 --- /dev/null +++ b/TODO @@ -0,0 +1,4 @@ +* find a better way to run "find" without creating the index storage, + if one is not already readily available (understand cost now, improve after merge) +* improve performance of splitlines +* think about cost of utf8 list strategy (Armin and CF) diff --git a/extra_tests/cffi_tests/cffi0/test_ownlib.py b/extra_tests/cffi_tests/cffi0/test_ownlib.py --- a/extra_tests/cffi_tests/cffi0/test_ownlib.py +++ b/extra_tests/cffi_tests/cffi0/test_ownlib.py @@ -352,6 +352,8 @@ def test_modify_struct_value(self): if self.module is None: py.test.skip("fix the auto-generation of the tiny test lib") + if self.Backend is CTypesBackend: + py.test.skip("fails with the ctypes backend on some architectures") ffi = FFI(backend=self.Backend()) ffi.cdef(""" typedef struct { diff --git a/extra_tests/cffi_tests/cffi1/test_pkgconfig.py b/extra_tests/cffi_tests/cffi1/test_pkgconfig.py new file mode 100644 --- /dev/null +++ b/extra_tests/cffi_tests/cffi1/test_pkgconfig.py @@ -0,0 +1,95 @@ +# Generated by pypy/tool/import_cffi.py +import sys +import subprocess +import py +import cffi.pkgconfig as pkgconfig +from cffi import PkgConfigError + + +def mock_call(libname, flag): + assert libname=="foobarbaz" + flags = { + "--cflags": "-I/usr/include/python3.6m -DABCD -DCFFI_TEST=1 -O42\n", + "--libs": "-L/usr/lib64 -lpython3.6 -shared\n", + } + return flags[flag] + + +def test_merge_flags(): + d1 = {"ham": [1, 2, 3], "spam" : ["a", "b", "c"], "foo" : []} + d2 = {"spam" : ["spam", "spam", "spam"], "bar" : ["b", "a", "z"]} + + pkgconfig.merge_flags(d1, d2) + assert d1 == { + "ham": [1, 2, 3], + "spam" : ["a", "b", "c", "spam", "spam", "spam"], + "bar" : ["b", "a", "z"], + "foo" : []} + + +def test_pkgconfig(): + assert pkgconfig.flags_from_pkgconfig([]) == {} + + saved = pkgconfig.call + try: + pkgconfig.call = mock_call + flags = pkgconfig.flags_from_pkgconfig(["foobarbaz"]) + finally: + pkgconfig.call = saved + assert flags == { + 'include_dirs': ['/usr/include/python3.6m'], + 'library_dirs': ['/usr/lib64'], + 'libraries': ['python3.6'], + 'define_macros': [('ABCD', None), ('CFFI_TEST', '1')], + 'extra_compile_args': ['-O42'], + 'extra_link_args': ['-shared'] + } + +class mock_subprocess: + PIPE = Ellipsis + class Popen: + def __init__(self, cmd, stdout, stderr): + if mock_subprocess.RESULT is None: + raise OSError("oops can't run") + assert cmd == ['pkg-config', '--print-errors', '--cflags', 'libfoo'] + def communicate(self): + bout, berr, rc = mock_subprocess.RESULT + self.returncode = rc + return bout, berr + +def test_call(): + saved = pkgconfig.subprocess + try: + pkgconfig.subprocess = mock_subprocess + + mock_subprocess.RESULT = None + e = py.test.raises(PkgConfigError, pkgconfig.call, "libfoo", "--cflags") + assert str(e.value) == "cannot run pkg-config: oops can't run" + + mock_subprocess.RESULT = b"", "Foo error!\n", 1 + e = py.test.raises(PkgConfigError, pkgconfig.call, "libfoo", "--cflags") + assert str(e.value) == "Foo error!" + + mock_subprocess.RESULT = b"abc\\def\n", "", 0 + e = py.test.raises(PkgConfigError, pkgconfig.call, "libfoo", "--cflags") + assert str(e.value).startswith("pkg-config --cflags libfoo returned an " + "unsupported backslash-escaped output:") + + mock_subprocess.RESULT = b"abc def\n", "", 0 + result = pkgconfig.call("libfoo", "--cflags") + assert result == "abc def\n" + + mock_subprocess.RESULT = b"abc def\n", "", 0 + result = pkgconfig.call("libfoo", "--cflags") + assert result == "abc def\n" + + if sys.version_info >= (3,): + mock_subprocess.RESULT = b"\xff\n", "", 0 + e = py.test.raises(PkgConfigError, pkgconfig.call, + "libfoo", "--cflags", encoding="utf-8") + assert str(e.value) == ( + "pkg-config --cflags libfoo returned bytes that cannot be " + "decoded with encoding 'utf-8':\nb'\\xff\\n'") + + finally: + pkgconfig.subprocess = saved diff --git a/extra_tests/cffi_tests/embedding/thread3-test.c b/extra_tests/cffi_tests/embedding/thread3-test.c --- a/extra_tests/cffi_tests/embedding/thread3-test.c +++ b/extra_tests/cffi_tests/embedding/thread3-test.c @@ -52,5 +52,6 @@ assert(status == 0); } printf("done\n"); + fflush(stdout); /* this is occasionally needed on Windows */ return 0; } diff --git a/extra_tests/ctypes_tests/test_structures.py b/extra_tests/ctypes_tests/test_structures.py --- a/extra_tests/ctypes_tests/test_structures.py +++ b/extra_tests/ctypes_tests/test_structures.py @@ -124,12 +124,15 @@ ms.n = 0xff00 return repr(ba[:]) + nstruct = dostruct(Native) if sys.byteorder == 'little': - assert dostruct(Native) == dostruct(Little) - assert dostruct(Native) != dostruct(Big) + assert nstruct == dostruct(Little) + assert nstruct != dostruct(Big) + assert Big._fields_[0][1] is not i else: - assert dostruct(Native) == dostruct(Big) - assert dostruct(Native) != dostruct(Little) + assert nstruct == dostruct(Big) + assert nstruct != dostruct(Little) + assert Little._fields_[0][1] is not i def test_from_buffer_copy(): from array import array @@ -190,3 +193,20 @@ assert sizeof(s) == 3 * sizeof(c_int) assert s.a == 4 # 256 + 4 assert s.b == -123 + +def test_memoryview(): + class S(Structure): + _fields_ = [('a', c_int16), + ('b', c_int16), + ] + + S3 = S * 3 + c_array = (2 * S3)( + S3(S(a=0, b=1), S(a=2, b=3), S(a=4, b=5)), + S3(S(a=6, b=7), S(a=8, b=9), S(a=10, b=11)), + ) + + mv = memoryview(c_array) + assert mv.format == 'T{ 0: - raise Error("newline inside string") - raise - - self.line_num += 1 - - if '\0' in line: - raise Error("line contains NULL byte") - pos = 0 - while pos < len(line): - pos = self._parse_process_char(line, pos) - self._parse_eol() - - if self.state == self.START_RECORD: - break - - fields = self.fields - self.fields = [] - return fields - - def _parse_process_char(self, line, pos): - c = line[pos] - if self.state == self.IN_FIELD: - # in unquoted field - pos2 = pos - while True: - if c in '\n\r': - # end of line - return [fields] - if pos2 > pos: - self._parse_add_char(line[pos:pos2]) - pos = pos2 - self._parse_save_field() - self.state = self.EAT_CRNL - elif c == self.dialect.escapechar: - # possible escaped character - pos2 -= 1 - self.state = self.ESCAPED_CHAR - elif c == self.dialect.delimiter: - # save field - wait for new field - if pos2 > pos: - self._parse_add_char(line[pos:pos2]) - pos = pos2 - self._parse_save_field() - self.state = self.START_FIELD - else: - # normal character - save in field - pos2 += 1 - if pos2 < len(line): - c = line[pos2] - continue - break - if pos2 > pos: - self._parse_add_char(line[pos:pos2]) - pos = pos2 - 1 - - elif self.state == self.START_RECORD: - if c in '\n\r': - self.state = self.EAT_CRNL - else: - self.state = self.START_FIELD - # restart process - self._parse_process_char(line, pos) - - elif self.state == self.START_FIELD: - if c in '\n\r': - # save empty field - return [fields] - self._parse_save_field() - self.state = self.EAT_CRNL - elif (c == self.dialect.quotechar - and self.dialect.quoting != QUOTE_NONE): - # start quoted field - self.state = self.IN_QUOTED_FIELD - elif c == self.dialect.escapechar: - # possible escaped character - self.state = self.ESCAPED_CHAR - elif c == ' ' and self.dialect.skipinitialspace: - # ignore space at start of field - pass - elif c == self.dialect.delimiter: - # save empty field - self._parse_save_field() - else: - # begin new unquoted field - if self.dialect.quoting == QUOTE_NONNUMERIC: - self.numeric_field = True - self._parse_add_char(c) - self.state = self.IN_FIELD - - elif self.state == self.ESCAPED_CHAR: - self._parse_add_char(c) - self.state = self.IN_FIELD - - elif self.state == self.IN_QUOTED_FIELD: - if c == self.dialect.escapechar: - # possible escape character - self.state = self.ESCAPE_IN_QUOTED_FIELD - elif (c == self.dialect.quotechar - and self.dialect.quoting != QUOTE_NONE): - if self.dialect.doublequote: - # doublequote; " represented by "" - self.state = self.QUOTE_IN_QUOTED_FIELD - else: - #end of quote part of field - self.state = self.IN_FIELD - else: - # normal character - save in field - self._parse_add_char(c) - - elif self.state == self.ESCAPE_IN_QUOTED_FIELD: - self._parse_add_char(c) - self.state = self.IN_QUOTED_FIELD - - elif self.state == self.QUOTE_IN_QUOTED_FIELD: - # doublequote - seen a quote in a quoted field - if (c == self.dialect.quotechar - and self.dialect.quoting != QUOTE_NONE): - # save "" as " - self._parse_add_char(c) - self.state = self.IN_QUOTED_FIELD - elif c == self.dialect.delimiter: - # save field - wait for new field - self._parse_save_field() - self.state = self.START_FIELD - elif c in '\r\n': - # end of line - return [fields] - self._parse_save_field() - self.state = self.EAT_CRNL - elif not self.dialect.strict: - self._parse_add_char(c) - self.state = self.IN_FIELD - else: - raise Error("'%c' expected after '%c'" % - (self.dialect.delimiter, self.dialect.quotechar)) - - elif self.state == self.EAT_CRNL: - if c not in '\r\n': - raise Error("new-line character seen in unquoted field - " - "do you need to open the file " - "in universal-newline mode?") - - else: - raise RuntimeError("unknown state: %r" % (self.state,)) - - return pos + 1 - - def _parse_eol(self): - if self.state == self.EAT_CRNL: - self.state = self.START_RECORD - elif self.state == self.START_RECORD: - # empty line - return [] - pass - elif self.state == self.IN_FIELD: - # in unquoted field - # end of line - return [fields] - self._parse_save_field() - self.state = self.START_RECORD - elif self.state == self.START_FIELD: - # save empty field - return [fields] - self._parse_save_field() - self.state = self.START_RECORD - elif self.state == self.ESCAPED_CHAR: - self._parse_add_char('\n') - self.state = self.IN_FIELD - elif self.state == self.IN_QUOTED_FIELD: - pass - elif self.state == self.ESCAPE_IN_QUOTED_FIELD: - self._parse_add_char('\n') - self.state = self.IN_QUOTED_FIELD - elif self.state == self.QUOTE_IN_QUOTED_FIELD: - # end of line - return [fields] - self._parse_save_field() - self.state = self.START_RECORD - else: - raise RuntimeError("unknown state: %r" % (self.state,)) - - def _parse_save_field(self): - field, self.field = self.field, '' - if self.numeric_field: - self.numeric_field = False - field = float(field) - self.fields.append(field) - - def _parse_add_char(self, c): - if len(self.field) + len(c) > _field_limit: - raise Error("field larger than field limit (%d)" % (_field_limit)) - self.field += c - - -class Writer(object): - """CSV writer - - Writer objects are responsible for generating tabular data - in CSV format from sequence input.""" - - def __init__(self, file, dialect=None, **kwargs): - if not (hasattr(file, 'write') and callable(file.write)): - raise TypeError("argument 1 must have a 'write' method") - self.writeline = file.write - self.dialect = _call_dialect(dialect, kwargs) - - def _join_reset(self): - self.rec = [] - self.num_fields = 0 - - def _join_append(self, field, quoted, quote_empty): - dialect = self.dialect - # If this is not the first field we need a field separator - if self.num_fields > 0: - self.rec.append(dialect.delimiter) - - if dialect.quoting == QUOTE_NONE: - need_escape = tuple(dialect.lineterminator) + ( - dialect.escapechar, # escapechar always first - dialect.delimiter, dialect.quotechar) - - else: - for c in tuple(dialect.lineterminator) + ( - dialect.delimiter, dialect.escapechar): - if c and c in field: - quoted = True - - need_escape = () - if dialect.quotechar in field: - if dialect.doublequote: - field = field.replace(dialect.quotechar, - dialect.quotechar * 2) - quoted = True - else: - need_escape = (dialect.quotechar,) - - - for c in need_escape: - if c and c in field: - if not dialect.escapechar: - raise Error("need to escape, but no escapechar set") - field = field.replace(c, dialect.escapechar + c) - - # If field is empty check if it needs to be quoted - if field == '' and quote_empty: - if dialect.quoting == QUOTE_NONE: - raise Error("single empty field record must be quoted") - quoted = 1 - - if quoted: - field = dialect.quotechar + field + dialect.quotechar - - self.rec.append(field) - self.num_fields += 1 - - - - def writerow(self, row): - dialect = self.dialect - try: - rowlen = len(row) - except TypeError: - raise Error("sequence expected") - - # join all fields in internal buffer - self._join_reset() - - for field in row: - quoted = False - if dialect.quoting == QUOTE_NONNUMERIC: - try: - float(field) - except: - quoted = True - # This changed since 2.5: - # quoted = not isinstance(field, (int, long, float)) - elif dialect.quoting == QUOTE_ALL: - quoted = True - - if field is None: - value = "" - elif isinstance(field, float): - value = repr(field) - else: - value = str(field) - self._join_append(value, quoted, rowlen == 1) - - # add line terminator - self.rec.append(dialect.lineterminator) - - self.writeline(''.join(self.rec)) - - def writerows(self, rows): - for row in rows: - self.writerow(row) - -def reader(*args, **kwargs): - """ - csv_reader = reader(iterable [, dialect='excel'] - [optional keyword args]) - for row in csv_reader: - process(row) - - The "iterable" argument can be any object that returns a line - of input for each iteration, such as a file object or a list. The - optional \"dialect\" parameter is discussed below. The function - also accepts optional keyword arguments which override settings - provided by the dialect. - - The returned object is an iterator. Each iteration returns a row - of the CSV file (which can span multiple input lines)""" - - return Reader(*args, **kwargs) - -def writer(*args, **kwargs): - """ - csv_writer = csv.writer(fileobj [, dialect='excel'] - [optional keyword args]) - for row in sequence: - csv_writer.writerow(row) - - [or] - - csv_writer = csv.writer(fileobj [, dialect='excel'] - [optional keyword args]) - csv_writer.writerows(rows) - - The \"fileobj\" argument can be any object that supports the file API.""" - return Writer(*args, **kwargs) - - -undefined = object() -def field_size_limit(limit=undefined): - """Sets an upper limit on parsed fields. - csv.field_size_limit([limit]) - - Returns old limit. If limit is not given, no new limit is set and - the old limit is returned""" - - global _field_limit - old_limit = _field_limit - - if limit is not undefined: - if not isinstance(limit, (int, long)): - raise TypeError("int expected, got %s" % - (limit.__class__.__name__,)) - _field_limit = limit - - return old_limit diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py --- a/lib_pypy/_ctypes/array.py +++ b/lib_pypy/_ctypes/array.py @@ -4,6 +4,7 @@ from _ctypes.basics import _CData, cdata_from_address, _CDataMeta, sizeof from _ctypes.basics import keepalive_key, store_reference, ensure_objects from _ctypes.basics import CArgObject, as_ffi_pointer +import sys, __pypy__, struct class ArrayMeta(_CDataMeta): def __new__(self, name, cls, typedict): @@ -241,6 +242,24 @@ def _as_ffi_pointer_(self, ffitype): return as_ffi_pointer(self, ffitype) + def __buffer__(self, flags): + shape = [] + obj = self + while 1: + shape.append(obj._length_) + try: + obj[0]._length_ + except (AttributeError, IndexError): + break + obj = obj[0] + + fmt = get_format_str(obj._type_) + try: + itemsize = struct.calcsize(fmt[1:]) + except: + itemsize = sizeof(obj[0]) + return __pypy__.newmemoryview(memoryview(self._buffer), itemsize, fmt, shape) + ARRAY_CACHE = {} def create_array_type(base, length): @@ -260,3 +279,31 @@ cls = ArrayMeta(name, (Array,), tpdict) ARRAY_CACHE[key] = cls return cls + +byteorder = {'little': '<', 'big': '>'} +swappedorder = {'little': '>', 'big': '<'} + +def get_format_str(typ): + if hasattr(typ, '_fields_'): + if hasattr(typ, '_swappedbytes_'): + bo = swappedorder[sys.byteorder] + else: + bo = byteorder[sys.byteorder] + flds = [] + for name, obj in typ._fields_: + # Trim off the leading '<' or '>' + ch = get_format_str(obj)[1:] + if (ch) == 'B': + flds.append(byteorder[sys.byteorder]) + else: + flds.append(bo) + flds.append(ch) + flds.append(':') + flds.append(name) + flds.append(':') + return 'T{' + ''.join(flds) + '}' + elif hasattr(typ, '_type_'): + ch = typ._type_ + return byteorder[sys.byteorder] + ch + else: + raise ValueError('cannot get format string for %r' % typ) diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py --- a/lib_pypy/_ctypes/basics.py +++ b/lib_pypy/_ctypes/basics.py @@ -2,8 +2,15 @@ from _rawffi import alt as _ffi import sys -try: from __pypy__ import builtinify -except ImportError: builtinify = lambda f: f +try: + from __pypy__ import builtinify +except ImportError: + builtinify = lambda f: f + +try: + from __pypy__.bufferable import bufferable +except ImportError: + bufferable = object keepalive_key = str # XXX fix this when provided with test @@ -64,7 +71,7 @@ 'resbuffer' is a _rawffi array of length 1 containing the value, and this returns a general Python object that corresponds. """ - res = object.__new__(self) + res = bufferable.__new__(self) res.__class__ = self res.__dict__['_buffer'] = resbuffer if base is not None: @@ -158,7 +165,7 @@ def __ne__(self, other): return self._obj != other -class _CData(object): +class _CData(bufferable): """ The most basic object for all ctypes types """ __metaclass__ = _CDataMeta diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py --- a/lib_pypy/_ctypes/pointer.py +++ b/lib_pypy/_ctypes/pointer.py @@ -7,8 +7,7 @@ from _ctypes.array import Array, array_get_slice_params, array_slice_getitem,\ array_slice_setitem -try: from __pypy__ import builtinify -except ImportError: builtinify = lambda f: f +from __pypy__ import builtinify, newmemoryview # This cache maps types to pointers to them. _pointer_type_cache = {} @@ -135,6 +134,9 @@ def _as_ffi_pointer_(self, ffitype): return as_ffi_pointer(self, ffitype) + def __buffer__(self, flags): + mv = memoryview(self.getcontents()) + return newmemoryview(mv, mv.itemsize, '&' + mv.format, mv.shape) def _cast_addr(obj, _, tp): if not (isinstance(tp, _CDataMeta) and tp._is_pointer_like()): diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -2,9 +2,9 @@ import _rawffi from _ctypes.basics import _CData, _CDataMeta, keepalive_key,\ store_reference, ensure_objects, CArgObject -from _ctypes.array import Array +from _ctypes.array import Array, get_format_str from _ctypes.pointer import _Pointer -import inspect +import inspect, __pypy__ def names_and_fields(self, _fields_, superclass, anonymous_fields=None): @@ -176,6 +176,11 @@ class StructOrUnionMeta(_CDataMeta): def __new__(self, name, cls, typedict): res = type.__new__(self, name, cls, typedict) + if hasattr(res, '_swappedbytes_') and '_fields_' in typedict: + # Activate the stdlib ctypes._swapped_meta.__setattr__ to convert fields + tmp = res._fields_ + delattr(res, '_fields_') + setattr(res, '_fields_', tmp) if "_abstract_" in typedict: return res cls = cls or (object,) @@ -254,17 +259,7 @@ or cls is union.Union): raise TypeError("abstract class") if hasattr(cls, '_swappedbytes_'): - fields = [None] * len(cls._fields_) - for i in range(len(cls._fields_)): - if cls._fields_[i][1] == cls._fields_[i][1].__dict__.get('__ctype_be__', None): - swapped = cls._fields_[i][1].__dict__.get('__ctype_le__', cls._fields_[i][1]) - else: - swapped = cls._fields_[i][1].__dict__.get('__ctype_be__', cls._fields_[i][1]) - if len(cls._fields_[i]) < 3: - fields[i] = (cls._fields_[i][0], swapped) - else: - fields[i] = (cls._fields_[i][0], swapped, cls._fields_[i][2]) - names_and_fields(cls, fields, _CData, cls.__dict__.get('_anonymous_', None)) + names_and_fields(cls, cls._fields_, _CData, cls.__dict__.get('_anonymous_', None)) self = super(_CData, cls).__new__(cls) if hasattr(cls, '_ffistruct_'): self.__dict__['_buffer'] = self._ffistruct_(autofree=True) @@ -304,6 +299,10 @@ def _to_ffi_param(self): return self._buffer + def __buffer__(self, flags): + fmt = get_format_str(self) + itemsize = type(self)._sizeofinstances() + return __pypy__.newmemoryview(memoryview(self._buffer), itemsize, fmt) class StructureMeta(StructOrUnionMeta): _is_union = False diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.12.0 +Version: 1.12.2 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -3,9 +3,10 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing +from .error import PkgConfigError -__version__ = "1.12.0" -__version_info__ = (1, 12, 0) +__version__ = "1.12.2" +__version_info__ = (1, 12, 2) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h --- a/lib_pypy/cffi/_cffi_include.h +++ b/lib_pypy/cffi/_cffi_include.h @@ -8,43 +8,20 @@ the same works for the other two macros. Py_DEBUG implies them, but not the other way around. - The implementation is messy (issue #350): on Windows, with _MSC_VER, - we have to define Py_LIMITED_API even before including pyconfig.h. - In that case, we guess what pyconfig.h will do to the macros above, - and check our guess after the #include. - - Note that on Windows, with CPython 3.x, you need virtualenv version - >= 16.0.0. Older versions don't copy PYTHON3.DLL. As a workaround - you can remove the definition of Py_LIMITED_API here. - - See also 'py_limited_api' in cffi/setuptools_ext.py. + Issue #350 is still open: on Windows, the code here causes it to link + with PYTHON36.DLL (for example) instead of PYTHON3.DLL. A fix was + attempted in 164e526a5515 and 14ce6985e1c3, but reverted: virtualenv + does not make PYTHON3.DLL available, and so the "correctly" compiled + version would not run inside a virtualenv. We will re-apply the fix + after virtualenv has been fixed for some time. For explanation, see + issue #355. For a workaround if you want PYTHON3.DLL and don't worry + about virtualenv, see issue #350. See also 'py_limited_api' in + setuptools_ext.py. */ #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) -# ifdef _MSC_VER -# if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) -# define Py_LIMITED_API -# endif -# include - /* sanity-check: Py_LIMITED_API will cause crashes if any of these - are also defined. Normally, the Python file PC/pyconfig.h does not - cause any of these to be defined, with the exception that _DEBUG - causes Py_DEBUG. Double-check that. */ -# ifdef Py_LIMITED_API -# if defined(Py_DEBUG) -# error "pyconfig.h unexpectedly defines Py_DEBUG, but Py_LIMITED_API is set" -# endif -# if defined(Py_TRACE_REFS) -# error "pyconfig.h unexpectedly defines Py_TRACE_REFS, but Py_LIMITED_API is set" -# endif -# if defined(Py_REF_DEBUG) -# error "pyconfig.h unexpectedly defines Py_REF_DEBUG, but Py_LIMITED_API is set" -# endif -# endif -# else -# include -# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) -# define Py_LIMITED_API -# endif +# include +# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API # endif #endif diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -221,7 +221,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.12.0" + "\ncompiled with cffi version: 1.12.2" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -592,7 +592,7 @@ if sys.platform == "win32": # we need 'libpypy-c.lib'. Current distributions of # pypy (>= 4.1) contain it as 'libs/python27.lib'. - pythonlib = "python27" + pythonlib = "python{0[0]}{0[1]}".format(sys.version_info) if hasattr(sys, 'prefix'): ensure('library_dirs', os.path.join(sys.prefix, 'libs')) else: @@ -643,6 +643,16 @@ self._assigned_source = (str(module_name), source, source_extension, kwds) + def set_source_pkgconfig(self, module_name, pkgconfig_libs, source, + source_extension='.c', **kwds): + from . import pkgconfig + if not isinstance(pkgconfig_libs, list): + raise TypeError("the pkgconfig_libs argument must be a list " + "of package names") + kwds2 = pkgconfig.flags_from_pkgconfig(pkgconfig_libs) + pkgconfig.merge_flags(kwds, kwds2) + self.set_source(module_name, source, source_extension, **kwds) + def distutils_extension(self, tmpdir='build', verbose=True): from distutils.dir_util import mkpath from .recompiler import recompile diff --git a/lib_pypy/cffi/error.py b/lib_pypy/cffi/error.py --- a/lib_pypy/cffi/error.py +++ b/lib_pypy/cffi/error.py @@ -1,8 +1,9 @@ class FFIError(Exception): - pass + __module__ = 'cffi' class CDefError(Exception): + __module__ = 'cffi' def __str__(self): try: current_decl = self.args[1] @@ -16,8 +17,15 @@ class VerificationError(Exception): """ An error raised when verification fails """ + __module__ = 'cffi' class VerificationMissing(Exception): """ An error raised when incomplete structures are passed into cdef, but no verification has been done """ + __module__ = 'cffi' + +class PkgConfigError(Exception): + """ An error raised for missing modules in pkg-config + """ + __module__ = 'cffi' diff --git a/lib_pypy/cffi/pkgconfig.py b/lib_pypy/cffi/pkgconfig.py new file mode 100644 --- /dev/null +++ b/lib_pypy/cffi/pkgconfig.py @@ -0,0 +1,121 @@ +# pkg-config, https://www.freedesktop.org/wiki/Software/pkg-config/ integration for cffi +import sys, os, subprocess + +from .error import PkgConfigError + + +def merge_flags(cfg1, cfg2): + """Merge values from cffi config flags cfg2 to cf1 + + Example: + merge_flags({"libraries": ["one"]}, {"libraries": ["two"]}) + {"libraries": ["one", "two"]} + """ + for key, value in cfg2.items(): + if key not in cfg1: + cfg1[key] = value + else: + if not isinstance(cfg1[key], list): + raise TypeError("cfg1[%r] should be a list of strings" % (key,)) + if not isinstance(value, list): + raise TypeError("cfg2[%r] should be a list of strings" % (key,)) + cfg1[key].extend(value) + return cfg1 + + +def call(libname, flag, encoding=sys.getfilesystemencoding()): + """Calls pkg-config and returns the output if found + """ + a = ["pkg-config", "--print-errors"] + a.append(flag) + a.append(libname) + try: + pc = subprocess.Popen(a, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except EnvironmentError as e: + raise PkgConfigError("cannot run pkg-config: %s" % (str(e).strip(),)) + + bout, berr = pc.communicate() + if pc.returncode != 0: + try: + berr = berr.decode(encoding) + except Exception: + pass + raise PkgConfigError(berr.strip()) + + if sys.version_info >= (3,) and not isinstance(bout, str): # Python 3.x + try: + bout = bout.decode(encoding) + except UnicodeDecodeError: + raise PkgConfigError("pkg-config %s %s returned bytes that cannot " + "be decoded with encoding %r:\n%r" % + (flag, libname, encoding, bout)) + + if os.altsep != '\\' and '\\' in bout: + raise PkgConfigError("pkg-config %s %s returned an unsupported " + "backslash-escaped output:\n%r" % + (flag, libname, bout)) + return bout + + +def flags_from_pkgconfig(libs): + r"""Return compiler line flags for FFI.set_source based on pkg-config output + + Usage + ... + ffibuilder.set_source("_foo", pkgconfig = ["libfoo", "libbar >= 1.8.3"]) + + If pkg-config is installed on build machine, then arguments include_dirs, + library_dirs, libraries, define_macros, extra_compile_args and + extra_link_args are extended with an output of pkg-config for libfoo and + libbar. + + Raises PkgConfigError in case the pkg-config call fails. + """ + + def get_include_dirs(string): + return [x[2:] for x in string.split() if x.startswith("-I")] + + def get_library_dirs(string): + return [x[2:] for x in string.split() if x.startswith("-L")] + + def get_libraries(string): + return [x[2:] for x in string.split() if x.startswith("-l")] + + # convert -Dfoo=bar to list of tuples [("foo", "bar")] expected by distutils + def get_macros(string): + def _macro(x): + x = x[2:] # drop "-D" + if '=' in x: + return tuple(x.split("=", 1)) # "-Dfoo=bar" => ("foo", "bar") + else: + return (x, None) # "-Dfoo" => ("foo", None) + return [_macro(x) for x in string.split() if x.startswith("-D")] + + def get_other_cflags(string): + return [x for x in string.split() if not x.startswith("-I") and + not x.startswith("-D")] + + def get_other_libs(string): + return [x for x in string.split() if not x.startswith("-L") and + not x.startswith("-l")] + + # return kwargs for given libname + def kwargs(libname): + fse = sys.getfilesystemencoding() + all_cflags = call(libname, "--cflags") + all_libs = call(libname, "--libs") + return { + "include_dirs": get_include_dirs(all_cflags), + "library_dirs": get_library_dirs(all_libs), + "libraries": get_libraries(all_libs), + "define_macros": get_macros(all_cflags), + "extra_compile_args": get_other_cflags(all_cflags), + "extra_link_args": get_other_libs(all_libs), + } + + # merge all arguments together + ret = {} + for libname in libs: + lib_flags = kwargs(libname) + merge_flags(ret, lib_flags) + return ret diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py --- a/lib_pypy/cffi/setuptools_ext.py +++ b/lib_pypy/cffi/setuptools_ext.py @@ -81,8 +81,14 @@ it doesn't so far, creating troubles. That's why we check for "not hasattr(sys, 'gettotalrefcount')" (the 2.7 compatible equivalent of 'd' not in sys.abiflags). (http://bugs.python.org/issue28401) + + On Windows, with CPython <= 3.4, it's better not to use py_limited_api + because virtualenv *still* doesn't copy PYTHON3.DLL on these versions. + For now we'll skip py_limited_api on all Windows versions to avoid an + inconsistent mess. """ - if 'py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount'): + if ('py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount') + and sys.platform != 'win32'): import setuptools try: setuptools_major_version = int(setuptools.__version__.partition('.')[0]) diff --git a/lib_pypy/pwd.py b/lib_pypy/pwd.py deleted file mode 100644 --- a/lib_pypy/pwd.py +++ /dev/null @@ -1,114 +0,0 @@ -# indirectly based on ctypes implementation: Victor Stinner, 2008-05-08 -""" -This module provides access to the Unix password database. -It is available on all Unix versions. - -Password database entries are reported as 7-tuples containing the following -items from the password database (see `'), in order: -pw_name, pw_passwd, pw_uid, pw_gid, pw_gecos, pw_dir, pw_shell. -The uid and gid items are integers, all others are strings. An -exception is raised if the entry asked for cannot be found. -""" - -from _pwdgrp_cffi import ffi, lib -import _structseq -import thread -_lock = thread.allocate_lock() - -try: from __pypy__ import builtinify -except ImportError: builtinify = lambda f: f - - -class struct_passwd: - """ - pwd.struct_passwd: Results from getpw*() routines. - - This object may be accessed either as a tuple of - (pw_name,pw_passwd,pw_uid,pw_gid,pw_gecos,pw_dir,pw_shell) - or via the object attributes as named in the above tuple. - """ - __metaclass__ = _structseq.structseqtype - name = "pwd.struct_passwd" - - pw_name = _structseq.structseqfield(0) - pw_passwd = _structseq.structseqfield(1) - pw_uid = _structseq.structseqfield(2) - pw_gid = _structseq.structseqfield(3) - pw_gecos = _structseq.structseqfield(4) - pw_dir = _structseq.structseqfield(5) - pw_shell = _structseq.structseqfield(6) - - -def _mkpwent(pw): - return struct_passwd([ - ffi.string(pw.pw_name), - ffi.string(pw.pw_passwd), - pw.pw_uid, - pw.pw_gid, - ffi.string(pw.pw_gecos), - ffi.string(pw.pw_dir), - ffi.string(pw.pw_shell)]) - - at builtinify -def getpwuid(uid): - """ - getpwuid(uid) -> (pw_name,pw_passwd,pw_uid, - pw_gid,pw_gecos,pw_dir,pw_shell) - Return the password database entry for the given numeric user ID. - See pwd.__doc__ for more on password database entries. - """ - with _lock: - pw = lib.getpwuid(uid) - if not pw: - raise KeyError("getpwuid(): uid not found: %s" % uid) - return _mkpwent(pw) - - at builtinify -def getpwnam(name): - """ - getpwnam(name) -> (pw_name,pw_passwd,pw_uid, - pw_gid,pw_gecos,pw_dir,pw_shell) - Return the password database entry for the given user name. - See pwd.__doc__ for more on password database entries. - """ - if not isinstance(name, basestring): - raise TypeError("expected string") - name = str(name) - with _lock: - pw = lib.getpwnam(name) - if not pw: - raise KeyError("getpwname(): name not found: %s" % name) - return _mkpwent(pw) - - at builtinify -def getpwall(): - """ - getpwall() -> list_of_entries - Return a list of all available password database entries, in arbitrary order. - See pwd.__doc__ for more on password database entries. - """ - users = [] - with _lock: - lib.setpwent() - while True: - pw = lib.getpwent() - if not pw: - break - users.append(_mkpwent(pw)) - lib.endpwent() - return users - -__all__ = ('struct_passwd', 'getpwuid', 'getpwnam', 'getpwall') - -if __name__ == "__main__": -# Uncomment next line to test CPython implementation -# from pwd import getpwuid, getpwnam, getpwall - from os import getuid - uid = getuid() - pw = getpwuid(uid) - print("uid %s: %s" % (pw.pw_uid, pw)) - name = pw.pw_name - print("name %r: %s" % (name, getpwnam(name))) - print("All:") - for pw in getpwall(): - print(pw) diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -12,11 +12,12 @@ except ImportError: pass else: - if __version__[:2] < '3.6': - s = settings(deadline=None) - settings.register_profile('default', s) - else: + try: settings.register_profile('default', deadline=None) + except Exception: + import warnings + warnings.warn("Version of hypothesis too old, " + "cannot set the deadline to None") settings.load_profile('default') # PyPy's command line extra options (these are added diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst --- a/pypy/doc/__pypy__-module.rst +++ b/pypy/doc/__pypy__-module.rst @@ -1,26 +1,186 @@ -.. comment: this document is very incomplete, should we generate - it automatically? +.. comment: this document may get out of synch with the code, but to generate + it automatically we would need to use pypy to run sphinx-build The ``__pypy__`` module ======================= The ``__pypy__`` module is the main entry point to special features provided -by PyPy's standard interpreter. Its content depends on :doc:`configuration options ` -which may add new functionality and functions whose existence or non-existence -indicates the presence of such features. - +by PyPy's standard interpreter. Its content depends on :doc:`configuration +options ` which may add new functionality and functions whose +existence or non-existence indicates the presence of such features. These are +generally used for compatibility when writing pure python modules that in +CPython are written in C. Not available in CPython, and so must be used inside a +``if platform.python_implementation == 'PyPy'`` block or otherwise hidden from +the CPython interpreter. Generally available functionality --------------------------------- - - ``internal_repr(obj)``: return the interpreter-level representation of an - object. - - ``bytebuffer(length)``: return a new read-write buffer of the given length. - It works like a simplified array of characters (actually, depending on the - configuration the ``array`` module internally uses this). - - ``attach_gdb()``: start a GDB at the interpreter-level (or a PDB before translation). + - ``internal_repr(obj)``: return the interpreter-level representation of an + object. + - ``bytebuffer(length)``: return a new read-write buffer of the given length. + It works like a simplified array of characters (actually, depending on the + configuration the ``array`` module internally uses this). + - ``attach_gdb()``: start a GDB at the interpreter-level (or a PDB before translation). + - ``newmemoryview(buffer, itemsize, format, shape=None, strides=None)``: + create a `memoryview` instance with the data from ``buffer`` and the + specified itemsize, format, and optional shape and strides. + + - ``bufferable``: a base class that provides a ``__buffer__(self, flags)`` + method for subclasses to override. This method should return a memoryview + instance of the class instance. It is called by the C-API's ``tp_as_buffer. + bf_getbuffer``. + + - ``builtinify(func)``: To implement at app-level modules that are, in CPython, + implemented in C: this decorator protects a function from being ever bound + like a method. Useful because some tests do things like put a "built-in" + function on a class and access it via the instance. + + - ``hidden_applevel(func)``: Decorator that hides a function's frame from + app-level + + - ``get_hidden_tb()``: Return the traceback of the current exception being + handled by a frame hidden from applevel. + + - ``lookup_special(obj, meth)``: Lookup up a special method on an object. + - ``do_what_I_mean`` + + - ``resizelist_hint(...)``: Reallocate the underlying storage of the argument + list to sizehint + + - ``newlist_hint(...)``: Create a new empty list that has an underlying + storage of length sizehint + + - ``add_memory_pressure(bytes)``: Add memory pressure of estimate bytes. + Useful when calling a C function that internally allocates a big chunk of + memory. This instructs the GC to garbage collect sooner than it would + otherwise. + + - ``newdict(type)``: Create a normal dict with a special implementation + strategy. ``type`` is a string and can be: + + * ``"module"`` - equivalent to ``some_module.__dict__`` + + * ``"instance"`` - equivalent to an instance dict with a not-changing-much + set of keys + + * ``"kwargs"`` - keyword args dict equivalent of what you get from + ``**kwargs`` in a function, optimized for passing around + + * ``"strdict"`` - string-key only dict. This one should be chosen + automatically + + - ``reversed_dict``: Enumerate the keys in a dictionary object in reversed + order. This is a ``__pypy__`` function instead of being simply done by + calling reversed(), for CPython compatibility: dictionaries are ordered in + PyPY but not in Cpython2.7. You should use the collections.OrderedDict + class for cases where ordering is important. That class implements + ``__reversed__`` by calling __pypy__.reversed_dict() + + - ``dict_popitem_first``: Interp-level implementation of + ``OrderedDict.popitem(last=False)``. + + - ``delitem_if_value_is`` Atomic equivalent to: ``if dict.get(key) is value: + del dict[key]``. + + SPECIAL USE CASES ONLY! Avoid using on dicts which are specialized, + e.g. to ``int`` or ``str`` keys, because it switches to the object + strategy. Also, the ``is`` operation is really pointer equality, so avoid + using it if ``value`` is an immutable object like ``int`` or ``str``. + + - ``move_to_end``: Move the key in a dictionary object into the first or last + position. This is used in Python 3.x to implement ``OrderedDict.move_to_end()``. + + - ``strategy(dict or list or set)``: Return the underlying strategy currently + used by the object + + - ``specialized_zip_2_lists`` + - ``locals_to_fast`` + - ``set_code_callback`` + - ``save_module_content_for_future_reload`` + - ``decode_long`` + - ``side_effects_ok``: For use with the reverse-debugger: this function + normally returns True, but will return False if we are evaluating a + debugging command like a watchpoint. You are responsible for not doing any + side effect at all (including no caching) when evaluating watchpoints. This + function is meant to help a bit---you can write:: + + if not __pypy__.side_effects_ok(): + skip the caching logic + + inside getter methods or properties, to make them usable from + watchpoints. Note that you need to re-run ``REVDB=.. pypy`` + after changing the Python code. + + - ``stack_almost_full``: Return True if the stack is more than 15/16th full. + - ``pyos_inputhook``: Call PyOS_InputHook() from the CPython C API + - ``os.real_getenv(...)`` gets OS environment variables skipping python code + - ``_pypydatetime`` provides base classes with correct C API interactions for + the pure-python ``datetime`` stdlib module + +Fast String Concatenation +------------------------- +Rather than in-place concatenation ``+=``, use these to enable fast, minimal +copy, string building. + + - ``builders.StringBuilder`` + - ``builders.UnicodeBuilder`` + +Interacting with the PyPy debug log +------------------------------------ + +The following functions can be used to write your own content to the +:ref:`PYPYLOG `. + + - ``debug_start(category, timestamp=False)``: open a new section; if + ``timestamp`` is ``True``, also return the timestamp which was written to + the log. + + - ``debug_stop(category, timestamp=False)``: close a section opened by + ``debug_start``. + + - ``debug_print(...)``: print arbitrary text to the log. + + - ``debug_print_once(category, ...)``: equivalent to ``debug_start`` + + ``debug_print`` + ``debug_stop``. + + - ``debug_flush``: flush the log. + + - ``debug_read_timestamp()``: read the timestamp from the same timer used by + the log. + + - ``debug_get_timestamp_unit()``: get the unit of the value returned by + ``debug_read_timestamp()``. + + +Depending on the architecture and operating system, PyPy uses different ways +to read timestamps, so the timestamps used in the log file are expressed in +varying units. It is possible to know which by calling +``debug_get_timestamp_unit()``, which can be one of the following values: + +``tsc`` + The default on ``x86`` machines: timestamps are expressed in CPU ticks, as + read by the `Time Stamp Counter`_. + +``ns`` + Timestamps are expressed in nanoseconds. + +``QueryPerformanceCounter`` + On Windows, in case for some reason ``tsc`` is not available: timestamps + are read using the win API ``QueryPerformanceCounter()``. + + +Unfortunately, there does not seem to be a reliable standard way for +converting ``tsc`` ticks into nanoseconds, although in practice on modern CPUs +it is enough to divide the ticks by the maximum nominal frequency of the CPU. +For this reason, PyPy gives the raw value, and leaves the job of doing the +conversion to external libraries. + +.. _`Time Stamp Counter`: https://en.wikipedia.org/wiki/Time_Stamp_Counter + + Transparent Proxy Functionality ------------------------------- @@ -34,6 +194,30 @@ its controller. Otherwise return None. +Additional Clocks for Timing +---------------------------- +The ``time`` submodule exposes the platform-dependent clock types such as +``CLOCK_BOOTTIME``, ``CLOCK_MONOTONIC``, ``CLOCK_MONOTONIC_COARSE``, +``CLOCK_MONOTONIC_RAW`` and two functions: + + - ``clock_gettime(m)`` which returns the clock type time in seconds and + - ``clock_getres(m)`` which returns the clock resolution in seconds. + +Extended Signal Handling +------------------------ +``thread.signals_enbaled`` is a context manager to use in non-main threads. + enables receiving signals in a "with" statement. More precisely, if a + signal is received by the process, then the signal handler might be + called either in the main thread (as usual) or within another thread + that is within a "with signals_enabled:". This other thread should be + ready to handle unexpected exceptions that the signal handler might + raise --- notably KeyboardInterrupt. + +Integer Operations with Overflow +-------------------------------- + - ``intop`` provides a module with integer operations that have + two-complement overflow behaviour instead of overflowing to longs + Functionality available on py.py (not after translation) -------------------------------------------------------- diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst --- a/pypy/doc/build.rst +++ b/pypy/doc/build.rst @@ -220,11 +220,12 @@ Making a debug build of PyPy ---------------------------- -If the Makefile is rerun with the lldebug or lldebug0 target, appropriate -compilation flags are added to add debug info and reduce compiler optimizations -to ``-O0`` respectively. If you stop in a debugger, you will see the -very wordy machine-generated C code from the rpython translation step, which -takes a little bit of reading to relate back to the rpython code. +Rerun the ``Makefile`` with the ``make lldebug`` or ``make lldebug0`` target, +which will build in a way that running under a debugger makes sense. +Appropriate compilation flags are added to add debug info, and for ``lldebug0`` +compiler optimizations are fully disabled. If you stop in a debugger, you will +see the very wordy machine-generated C code from the rpython translation step, +which takes a little bit of reading to relate back to the rpython code. Build cffi import libraries for the stdlib ------------------------------------------ diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -71,9 +71,9 @@ # module/cpyext/include/patchlevel.h # # The short X.Y version. -version = '7.1' +version = '7.2' # The full version, including alpha/beta/rc tags. -release = '7.1.0' +release = '7.2.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -90,7 +90,9 @@ Wenzhu Man Konstantin Lopuhin John Witulski + Stefan Beyer Jeremy Thurgood + Andrew Lawrence Greg Price Ivan Sichmann Freitas Dario Bertini @@ -101,7 +103,6 @@ Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov - Stefan Beyer William Leslie Paweł Piotr Przeradowski marky1991 @@ -119,6 +120,7 @@ Wanja Saatkamp Mike Blume Gerald Klix + Julian Berman Oscar Nierstrasz Rami Chowdhury Stefan H. Muller @@ -141,6 +143,7 @@ Anton Gulenko Sergey Matyunin Andrew Chambers + Łukasz Langa Nicolas Chauvat Andrew Durdin Ben Young @@ -263,7 +266,6 @@ Bobby Impollonia Roberto De Ioris Jeong YunWon - andrewjlawrence Christopher Armstrong Aaron Tubbs Vasantha Ganesh K @@ -295,7 +297,6 @@ Ben Darnell Juan Francisco Cantero Hurtado Godefroid Chappelle - Julian Berman Stephan Busemann Dan Colish timo diff --git a/pypy/doc/extending.rst b/pypy/doc/extending.rst --- a/pypy/doc/extending.rst +++ b/pypy/doc/extending.rst @@ -45,16 +45,13 @@ with the `CPython ctypes`_ version. It works for large examples, such as pyglet. PyPy's implementation is not strictly 100% compatible with CPython, but close enough for most cases. - -We also used to provide ``ctypes-configure`` for some API-level access. -This is now viewed as a precursor of CFFI, which you should use instead. More (but older) information is available :doc:`here `. Also, ctypes' performance is not as good as CFFI's. .. _CPython ctypes: http://docs.python.org/library/ctypes.html PyPy implements ctypes as pure Python code around two built-in modules -called ``_ffi`` and ``_rawffi``, which give a very low-level binding to +called ``_rawffi`` and ``_rawffi.alt``, which give a very low-level binding to the C library libffi_. Nowadays it is not recommended to use directly these two modules. diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -203,7 +203,7 @@ ``duration`` The total time spent inside minor collections since the last hook - call. See below for more information on the unit. + call, in seconds. ``duration_min`` The duration of the fastest minor collection since the last hook call. @@ -265,30 +265,6 @@ ``gc-collect-done`` is used only to give additional stats, but doesn't do any actual work. -A note about the ``duration`` field: depending on the architecture and -operating system, PyPy uses different ways to read timestamps, so ``duration`` -is expressed in varying units. It is possible to know which by calling -``__pypy__.debug_get_timestamp_unit()``, which can be one of the following -values: - -``tsc`` - The default on ``x86`` machines: timestamps are expressed in CPU ticks, as - read by the `Time Stamp Counter`_. - -``ns`` - Timestamps are expressed in nanoseconds. - -``QueryPerformanceCounter`` - On Windows, in case for some reason ``tsc`` is not available: timestamps - are read using the win API ``QueryPerformanceCounter()``. - - -Unfortunately, there does not seem to be a reliable standard way for -converting ``tsc`` ticks into nanoseconds, although in practice on modern CPUs -it is enough to divide the ticks by the maximum nominal frequency of the CPU. -For this reason, PyPy gives the raw value, and leaves the job of doing the -conversion to external libraries. - Here is an example of GC hooks in use:: import sys @@ -321,8 +297,6 @@ lst = [lst, 1, 2, 3] -.. _`Time Stamp Counter`: https://en.wikipedia.org/wiki/Time_Stamp_Counter - .. _minimark-environment-variables: Environment variables diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -1,11 +1,12 @@ Historical release notes ======================== -CPython 2.7 compatible versions -------------------------------- +Combined releases +----------------- .. toctree:: + release-v7.1.0.rst release-v7.0.0.rst release-v6.0.0.rst release-v5.10.1.rst @@ -14,6 +15,12 @@ release-v5.8.0.rst release-v5.7.1.rst release-v5.7.0.rst + +CPython 2.7 compatible versions +------------------------------- + +.. toctree:: + release-pypy2.7-v5.6.0.rst release-pypy2.7-v5.4.1.rst release-pypy2.7-v5.4.0.rst @@ -61,15 +68,6 @@ release-0.7.0.rst release-0.6 -CPython 3.5 compatible versions -------------------------------- - -.. toctree:: - - release-v5.8.0.rst - release-v5.7.1.rst - release-v5.7.0.rst - CPython 3.3 compatible versions ------------------------------- diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst --- a/pypy/doc/index.rst +++ b/pypy/doc/index.rst @@ -103,7 +103,7 @@ the `development mailing list`_. .. _#pypy on irc.freenode.net: irc://irc.freenode.net/pypy -.. _here: https://botbot.me/freenode/pypy/ +.. _here: https://quodlibet.duckdns.org/irc/pypy/latest.log.html#irc-end .. _Development mailing list: http://mail.python.org/mailman/listinfo/pypy-dev .. _Commit mailing list: http://mail.python.org/mailman/listinfo/pypy-commit .. _Development bug/feature tracker: https://bitbucket.org/pypy/pypy/issues diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst --- a/pypy/doc/man/pypy.1.rst +++ b/pypy/doc/man/pypy.1.rst @@ -99,6 +99,8 @@ If set, equivalent to the ``-W`` option (warning control). The value should be a comma-separated list of ``-W`` parameters. +.. _pypylog: + ``PYPYLOG`` If set to a non-empty value, enable logging, the format is: diff --git a/pypy/doc/release-v7.0.0.rst b/pypy/doc/release-v7.0.0.rst --- a/pypy/doc/release-v7.0.0.rst +++ b/pypy/doc/release-v7.0.0.rst @@ -19,10 +19,12 @@ Until we can work with downstream providers to distribute builds with PyPy, we have made packages for some common packages `available as wheels`_. -The GC now has `hooks`_ to gain more insights into its performance, and it is -now possible to manually manage the GC by using a combination of -``gc.disable`` and ``gc.collect_step``. See the `GC blog post`_. +The `GC hooks`_ , which can be used to gain more insights into its +performance, has been improved and it is now possible to manually manage the +GC by using a combination of ``gc.disable`` and ``gc.collect_step``. See the +`GC blog post`_. +.. _`GC hooks`: http://doc.pypy.org/en/latest/gc_info.html#semi-manual-gc-management We updated the `cffi`_ module included in PyPy to version 1.12, and the `cppyy`_ backend to 1.4. Please use these to wrap your C and C++ code, @@ -38,7 +40,7 @@ The utf8 branch that changes internal representation of unicode to utf8 did not make it into the release, so there is still more goodness coming. -You can download the v6.0 releases here: +You can download the v7.0 releases here: http://pypy.org/download.html @@ -48,7 +50,7 @@ We would also like to thank our contributors and encourage new people to join the project. PyPy has many layers and we need help with all of them: `PyPy`_ -and `RPython`_ documentation improvements, tweaking popular `modules`_ to run +and `RPython`_ documentation improvements, tweaking popular modules to run on pypy, or general `help`_ with making RPython's JIT even better. .. _`PyPy`: index.html diff --git a/pypy/doc/release-v7.1.0.rst b/pypy/doc/release-v7.1.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v7.1.0.rst @@ -0,0 +1,84 @@ +========================================= +PyPy v7.1.0: release of 2.7, and 3.6-beta +========================================= + +The PyPy team is proud to release the version 7.1.0 of PyPy, which includes +two different interpreters: + + - PyPy2.7, which is an interpreter supporting the syntax and the features of + Python 2.7 From pypy.commits at gmail.com Tue Mar 12 15:14:08 2019 From: pypy.commits at gmail.com (rlamy) Date: Tue, 12 Mar 2019 12:14:08 -0700 (PDT) Subject: [pypy-commit] pypy apptest-file: fix merge Message-ID: <5c880500.1c69fb81.d3446.3ab0@mx.google.com> Author: Ronan Lamy Branch: apptest-file Changeset: r96299:2c2298a84910 Date: 2019-03-12 18:23 +0000 http://bitbucket.org/pypy/pypy/changeset/2c2298a84910/ Log: fix merge diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -6,7 +6,7 @@ from rpython.rlib import rutf8 from rpython.rlib.rarithmetic import r_uint, intmask from rpython.rtyper.lltypesystem import rffi -from pypy.module.unicodedata import unicodedb +from pypy.module.unicodedata.interp_ucd import unicodedb @specialize.memo() def decode_error_handler(space): @@ -233,7 +233,7 @@ slen = len(s) res = runicode.unicode_encode_mbcs(s, slen, errors, errorhandler) return res - + def str_decode_mbcs(s, errors, final, errorhandler): from rpython.rlib import runicode slen = len(s) From pypy.commits at gmail.com Tue Mar 12 15:14:06 2019 From: pypy.commits at gmail.com (rlamy) Date: Tue, 12 Mar 2019 12:14:06 -0700 (PDT) Subject: [pypy-commit] pypy py3tests: hg merge py3.6 Message-ID: <5c8804fe.1c69fb81.80589.210d@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r96298:baa5e3964d15 Date: 2019-03-12 18:28 +0000 http://bitbucket.org/pypy/pypy/changeset/baa5e3964d15/ Log: hg merge py3.6 diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -123,7 +123,9 @@ Wenzhu Man Konstantin Lopuhin John Witulski + Stefan Beyer Jeremy Thurgood + Andrew Lawrence Greg Price Ivan Sichmann Freitas Dario Bertini @@ -134,7 +136,6 @@ Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov - Stefan Beyer William Leslie Paweł Piotr Przeradowski marky1991 @@ -152,6 +153,7 @@ Wanja Saatkamp Mike Blume Gerald Klix + Julian Berman Oscar Nierstrasz Rami Chowdhury Stefan H. Muller @@ -174,6 +176,7 @@ Anton Gulenko Sergey Matyunin Andrew Chambers + Łukasz Langa Nicolas Chauvat Andrew Durdin Ben Young @@ -296,7 +299,6 @@ Bobby Impollonia Roberto De Ioris Jeong YunWon - andrewjlawrence Christopher Armstrong Aaron Tubbs Vasantha Ganesh K @@ -328,7 +330,6 @@ Ben Darnell Juan Francisco Cantero Hurtado Godefroid Chappelle - Julian Berman Stephan Busemann Dan Colish timo diff --git a/extra_tests/ctypes_tests/test_structures.py b/extra_tests/ctypes_tests/test_structures.py --- a/extra_tests/ctypes_tests/test_structures.py +++ b/extra_tests/ctypes_tests/test_structures.py @@ -119,12 +119,15 @@ ms.n = 0xff00 return repr(ba[:]) + nstruct = dostruct(Native) if sys.byteorder == 'little': - assert dostruct(Native) == dostruct(Little) - assert dostruct(Native) != dostruct(Big) + assert nstruct == dostruct(Little) + assert nstruct != dostruct(Big) + assert Big._fields_[0][1] is not i else: - assert dostruct(Native) == dostruct(Big) - assert dostruct(Native) != dostruct(Little) + assert nstruct == dostruct(Big) + assert nstruct != dostruct(Little) + assert Little._fields_[0][1] is not i def test_from_buffer_copy(): from array import array @@ -185,3 +188,20 @@ assert sizeof(s) == 3 * sizeof(c_int) assert s.a == 4 # 256 + 4 assert s.b == -123 + +def test_memoryview(): + class S(Structure): + _fields_ = [('a', c_int16), + ('b', c_int16), + ] + + S3 = S * 3 + c_array = (2 * S3)( + S3(S(a=0, b=1), S(a=2, b=3), S(a=4, b=5)), + S3(S(a=6, b=7), S(a=8, b=9), S(a=10, b=11)), + ) + + mv = memoryview(c_array) + assert mv.format == 'T{'} +swappedorder = {'little': '>', 'big': '<'} + +def get_format_str(typ): + if hasattr(typ, '_fields_'): + if hasattr(typ, '_swappedbytes_'): + bo = swappedorder[sys.byteorder] + else: + bo = byteorder[sys.byteorder] + flds = [] + for name, obj in typ._fields_: + # Trim off the leading '<' or '>' + ch = get_format_str(obj)[1:] + if (ch) == 'B': + flds.append(byteorder[sys.byteorder]) + else: + flds.append(bo) + flds.append(ch) + flds.append(':') + flds.append(name) + flds.append(':') + return 'T{' + ''.join(flds) + '}' + elif hasattr(typ, '_type_'): + ch = typ._type_ + return byteorder[sys.byteorder] + ch + else: + raise ValueError('cannot get format string for %r' % typ) diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py --- a/lib_pypy/_ctypes/basics.py +++ b/lib_pypy/_ctypes/basics.py @@ -2,8 +2,15 @@ from _rawffi import alt as _ffi import sys -try: from __pypy__ import builtinify -except ImportError: builtinify = lambda f: f +try: + from __pypy__ import builtinify +except ImportError: + builtinify = lambda f: f + +try: + from __pypy__.bufferable import bufferable +except ImportError: + bufferable = object keepalive_key = str # XXX fix this when provided with test @@ -64,7 +71,7 @@ 'resbuffer' is a _rawffi array of length 1 containing the value, and this returns a general Python object that corresponds. """ - res = object.__new__(self) + res = bufferable.__new__(self) res.__class__ = self res.__dict__['_buffer'] = resbuffer if base is not None: @@ -148,7 +155,7 @@ def __ne__(self, other): return self._obj != other -class _CData(object, metaclass=_CDataMeta): +class _CData(bufferable, metaclass=_CDataMeta): """ The most basic object for all ctypes types """ _objects = None diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py --- a/lib_pypy/_ctypes/pointer.py +++ b/lib_pypy/_ctypes/pointer.py @@ -7,8 +7,7 @@ from _ctypes.array import Array, array_get_slice_params, array_slice_getitem,\ array_slice_setitem -try: from __pypy__ import builtinify -except ImportError: builtinify = lambda f: f +from __pypy__ import builtinify, newmemoryview # This cache maps types to pointers to them. _pointer_type_cache = {} @@ -134,6 +133,9 @@ def _as_ffi_pointer_(self, ffitype): return as_ffi_pointer(self, ffitype) + def __buffer__(self, flags): + mv = memoryview(self.getcontents()) + return newmemoryview(mv, mv.itemsize, '&' + mv.format, mv.shape) def _cast_addr(obj, _, tp): if not (isinstance(tp, _CDataMeta) and tp._is_pointer_like()): diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -2,9 +2,9 @@ import _rawffi from _ctypes.basics import _CData, _CDataMeta, keepalive_key,\ store_reference, ensure_objects, CArgObject -from _ctypes.array import Array +from _ctypes.array import Array, get_format_str from _ctypes.pointer import _Pointer -import inspect +import inspect, __pypy__ def names_and_fields(self, _fields_, superclass, anonymous_fields=None): @@ -176,6 +176,11 @@ class StructOrUnionMeta(_CDataMeta): def __new__(self, name, cls, typedict): res = type.__new__(self, name, cls, typedict) + if hasattr(res, '_swappedbytes_') and '_fields_' in typedict: + # Activate the stdlib ctypes._swapped_meta.__setattr__ to convert fields + tmp = res._fields_ + delattr(res, '_fields_') + setattr(res, '_fields_', tmp) if "_abstract_" in typedict: return res cls = cls or (object,) @@ -253,17 +258,7 @@ or cls is union.Union): raise TypeError("abstract class") if hasattr(cls, '_swappedbytes_'): - fields = [None] * len(cls._fields_) - for i in range(len(cls._fields_)): - if cls._fields_[i][1] == cls._fields_[i][1].__dict__.get('__ctype_be__', None): - swapped = cls._fields_[i][1].__dict__.get('__ctype_le__', cls._fields_[i][1]) - else: - swapped = cls._fields_[i][1].__dict__.get('__ctype_be__', cls._fields_[i][1]) - if len(cls._fields_[i]) < 3: - fields[i] = (cls._fields_[i][0], swapped) - else: - fields[i] = (cls._fields_[i][0], swapped, cls._fields_[i][2]) - names_and_fields(cls, fields, _CData, cls.__dict__.get('_anonymous_', None)) + names_and_fields(cls, cls._fields_, _CData, cls.__dict__.get('_anonymous_', None)) self = super(_CData, cls).__new__(cls) if hasattr(cls, '_ffistruct_'): self.__dict__['_buffer'] = self._ffistruct_(autofree=True) @@ -303,6 +298,10 @@ def _to_ffi_param(self): return self._buffer + def __buffer__(self, flags): + fmt = get_format_str(self) + itemsize = type(self)._sizeofinstances() + return __pypy__.newmemoryview(memoryview(self._buffer), itemsize, fmt) class StructureMeta(StructOrUnionMeta): _is_union = False diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.12.1 +Version: 1.12.2 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -5,8 +5,8 @@ from .error import CDefError, FFIError, VerificationError, VerificationMissing from .error import PkgConfigError -__version__ = "1.12.1" -__version_info__ = (1, 12, 1) +__version__ = "1.12.2" +__version_info__ = (1, 12, 2) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -221,7 +221,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.12.1" + "\ncompiled with cffi version: 1.12.2" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst --- a/pypy/doc/__pypy__-module.rst +++ b/pypy/doc/__pypy__-module.rst @@ -1,26 +1,186 @@ -.. comment: this document is very incomplete, should we generate - it automatically? +.. comment: this document may get out of synch with the code, but to generate + it automatically we would need to use pypy to run sphinx-build The ``__pypy__`` module ======================= The ``__pypy__`` module is the main entry point to special features provided -by PyPy's standard interpreter. Its content depends on :doc:`configuration options ` -which may add new functionality and functions whose existence or non-existence -indicates the presence of such features. - +by PyPy's standard interpreter. Its content depends on :doc:`configuration +options ` which may add new functionality and functions whose +existence or non-existence indicates the presence of such features. These are +generally used for compatibility when writing pure python modules that in +CPython are written in C. Not available in CPython, and so must be used inside a +``if platform.python_implementation == 'PyPy'`` block or otherwise hidden from +the CPython interpreter. Generally available functionality --------------------------------- - - ``internal_repr(obj)``: return the interpreter-level representation of an - object. - - ``bytebuffer(length)``: return a new read-write buffer of the given length. - It works like a simplified array of characters (actually, depending on the - configuration the ``array`` module internally uses this). - - ``attach_gdb()``: start a GDB at the interpreter-level (or a PDB before translation). + - ``internal_repr(obj)``: return the interpreter-level representation of an + object. + - ``bytebuffer(length)``: return a new read-write buffer of the given length. + It works like a simplified array of characters (actually, depending on the + configuration the ``array`` module internally uses this). + - ``attach_gdb()``: start a GDB at the interpreter-level (or a PDB before translation). + - ``newmemoryview(buffer, itemsize, format, shape=None, strides=None)``: + create a `memoryview` instance with the data from ``buffer`` and the + specified itemsize, format, and optional shape and strides. + + - ``bufferable``: a base class that provides a ``__buffer__(self, flags)`` + method for subclasses to override. This method should return a memoryview + instance of the class instance. It is called by the C-API's ``tp_as_buffer. + bf_getbuffer``. + + - ``builtinify(func)``: To implement at app-level modules that are, in CPython, + implemented in C: this decorator protects a function from being ever bound + like a method. Useful because some tests do things like put a "built-in" + function on a class and access it via the instance. + + - ``hidden_applevel(func)``: Decorator that hides a function's frame from + app-level + + - ``get_hidden_tb()``: Return the traceback of the current exception being + handled by a frame hidden from applevel. + + - ``lookup_special(obj, meth)``: Lookup up a special method on an object. + - ``do_what_I_mean`` + + - ``resizelist_hint(...)``: Reallocate the underlying storage of the argument + list to sizehint + + - ``newlist_hint(...)``: Create a new empty list that has an underlying + storage of length sizehint + + - ``add_memory_pressure(bytes)``: Add memory pressure of estimate bytes. + Useful when calling a C function that internally allocates a big chunk of + memory. This instructs the GC to garbage collect sooner than it would + otherwise. + + - ``newdict(type)``: Create a normal dict with a special implementation + strategy. ``type`` is a string and can be: + + * ``"module"`` - equivalent to ``some_module.__dict__`` + + * ``"instance"`` - equivalent to an instance dict with a not-changing-much + set of keys + + * ``"kwargs"`` - keyword args dict equivalent of what you get from + ``**kwargs`` in a function, optimized for passing around + + * ``"strdict"`` - string-key only dict. This one should be chosen + automatically + + - ``reversed_dict``: Enumerate the keys in a dictionary object in reversed + order. This is a ``__pypy__`` function instead of being simply done by + calling reversed(), for CPython compatibility: dictionaries are ordered in + PyPY but not in Cpython2.7. You should use the collections.OrderedDict + class for cases where ordering is important. That class implements + ``__reversed__`` by calling __pypy__.reversed_dict() + + - ``dict_popitem_first``: Interp-level implementation of + ``OrderedDict.popitem(last=False)``. + + - ``delitem_if_value_is`` Atomic equivalent to: ``if dict.get(key) is value: + del dict[key]``. + + SPECIAL USE CASES ONLY! Avoid using on dicts which are specialized, + e.g. to ``int`` or ``str`` keys, because it switches to the object + strategy. Also, the ``is`` operation is really pointer equality, so avoid + using it if ``value`` is an immutable object like ``int`` or ``str``. + + - ``move_to_end``: Move the key in a dictionary object into the first or last + position. This is used in Python 3.x to implement ``OrderedDict.move_to_end()``. + + - ``strategy(dict or list or set)``: Return the underlying strategy currently + used by the object + + - ``specialized_zip_2_lists`` + - ``locals_to_fast`` + - ``set_code_callback`` + - ``save_module_content_for_future_reload`` + - ``decode_long`` + - ``side_effects_ok``: For use with the reverse-debugger: this function + normally returns True, but will return False if we are evaluating a + debugging command like a watchpoint. You are responsible for not doing any + side effect at all (including no caching) when evaluating watchpoints. This + function is meant to help a bit---you can write:: + + if not __pypy__.side_effects_ok(): + skip the caching logic + + inside getter methods or properties, to make them usable from + watchpoints. Note that you need to re-run ``REVDB=.. pypy`` + after changing the Python code. + + - ``stack_almost_full``: Return True if the stack is more than 15/16th full. + - ``pyos_inputhook``: Call PyOS_InputHook() from the CPython C API + - ``os.real_getenv(...)`` gets OS environment variables skipping python code + - ``_pypydatetime`` provides base classes with correct C API interactions for + the pure-python ``datetime`` stdlib module + +Fast String Concatenation +------------------------- +Rather than in-place concatenation ``+=``, use these to enable fast, minimal +copy, string building. + + - ``builders.StringBuilder`` + - ``builders.UnicodeBuilder`` + +Interacting with the PyPy debug log +------------------------------------ + +The following functions can be used to write your own content to the +:ref:`PYPYLOG `. + + - ``debug_start(category, timestamp=False)``: open a new section; if + ``timestamp`` is ``True``, also return the timestamp which was written to + the log. + + - ``debug_stop(category, timestamp=False)``: close a section opened by + ``debug_start``. + + - ``debug_print(...)``: print arbitrary text to the log. + + - ``debug_print_once(category, ...)``: equivalent to ``debug_start`` + + ``debug_print`` + ``debug_stop``. + + - ``debug_flush``: flush the log. + + - ``debug_read_timestamp()``: read the timestamp from the same timer used by + the log. + + - ``debug_get_timestamp_unit()``: get the unit of the value returned by + ``debug_read_timestamp()``. + + +Depending on the architecture and operating system, PyPy uses different ways +to read timestamps, so the timestamps used in the log file are expressed in +varying units. It is possible to know which by calling +``debug_get_timestamp_unit()``, which can be one of the following values: + +``tsc`` + The default on ``x86`` machines: timestamps are expressed in CPU ticks, as + read by the `Time Stamp Counter`_. + +``ns`` + Timestamps are expressed in nanoseconds. + +``QueryPerformanceCounter`` + On Windows, in case for some reason ``tsc`` is not available: timestamps + are read using the win API ``QueryPerformanceCounter()``. + + +Unfortunately, there does not seem to be a reliable standard way for +converting ``tsc`` ticks into nanoseconds, although in practice on modern CPUs +it is enough to divide the ticks by the maximum nominal frequency of the CPU. +For this reason, PyPy gives the raw value, and leaves the job of doing the +conversion to external libraries. + +.. _`Time Stamp Counter`: https://en.wikipedia.org/wiki/Time_Stamp_Counter + + Transparent Proxy Functionality ------------------------------- @@ -34,6 +194,30 @@ its controller. Otherwise return None. +Additional Clocks for Timing +---------------------------- +The ``time`` submodule exposes the platform-dependent clock types such as +``CLOCK_BOOTTIME``, ``CLOCK_MONOTONIC``, ``CLOCK_MONOTONIC_COARSE``, +``CLOCK_MONOTONIC_RAW`` and two functions: + + - ``clock_gettime(m)`` which returns the clock type time in seconds and + - ``clock_getres(m)`` which returns the clock resolution in seconds. + +Extended Signal Handling +------------------------ +``thread.signals_enbaled`` is a context manager to use in non-main threads. + enables receiving signals in a "with" statement. More precisely, if a + signal is received by the process, then the signal handler might be + called either in the main thread (as usual) or within another thread + that is within a "with signals_enabled:". This other thread should be + ready to handle unexpected exceptions that the signal handler might + raise --- notably KeyboardInterrupt. + +Integer Operations with Overflow +-------------------------------- + - ``intop`` provides a module with integer operations that have + two-complement overflow behaviour instead of overflowing to longs + Functionality available on py.py (not after translation) -------------------------------------------------------- diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -71,9 +71,9 @@ # module/cpyext/include/patchlevel.h # # The short X.Y version. -version = '7.1' +version = '7.2' # The full version, including alpha/beta/rc tags. -release = '7.1.0' +release = '7.2.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -90,7 +90,9 @@ Wenzhu Man Konstantin Lopuhin John Witulski + Stefan Beyer Jeremy Thurgood + Andrew Lawrence Greg Price Ivan Sichmann Freitas Dario Bertini @@ -101,7 +103,6 @@ Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov - Stefan Beyer William Leslie Paweł Piotr Przeradowski marky1991 @@ -119,6 +120,7 @@ Wanja Saatkamp Mike Blume Gerald Klix + Julian Berman Oscar Nierstrasz Rami Chowdhury Stefan H. Muller @@ -141,6 +143,7 @@ Anton Gulenko Sergey Matyunin Andrew Chambers + Łukasz Langa Nicolas Chauvat Andrew Durdin Ben Young @@ -263,7 +266,6 @@ Bobby Impollonia Roberto De Ioris Jeong YunWon - andrewjlawrence Christopher Armstrong Aaron Tubbs Vasantha Ganesh K @@ -295,7 +297,6 @@ Ben Darnell Juan Francisco Cantero Hurtado Godefroid Chappelle - Julian Berman Stephan Busemann Dan Colish timo diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -203,7 +203,7 @@ ``duration`` The total time spent inside minor collections since the last hook - call. See below for more information on the unit. + call, in seconds. ``duration_min`` The duration of the fastest minor collection since the last hook call. @@ -265,30 +265,6 @@ ``gc-collect-done`` is used only to give additional stats, but doesn't do any actual work. -A note about the ``duration`` field: depending on the architecture and -operating system, PyPy uses different ways to read timestamps, so ``duration`` -is expressed in varying units. It is possible to know which by calling -``__pypy__.debug_get_timestamp_unit()``, which can be one of the following -values: - -``tsc`` - The default on ``x86`` machines: timestamps are expressed in CPU ticks, as - read by the `Time Stamp Counter`_. - -``ns`` - Timestamps are expressed in nanoseconds. - -``QueryPerformanceCounter`` - On Windows, in case for some reason ``tsc`` is not available: timestamps - are read using the win API ``QueryPerformanceCounter()``. - - -Unfortunately, there does not seem to be a reliable standard way for -converting ``tsc`` ticks into nanoseconds, although in practice on modern CPUs -it is enough to divide the ticks by the maximum nominal frequency of the CPU. -For this reason, PyPy gives the raw value, and leaves the job of doing the -conversion to external libraries. - Here is an example of GC hooks in use:: import sys @@ -321,8 +297,6 @@ lst = [lst, 1, 2, 3] -.. _`Time Stamp Counter`: https://en.wikipedia.org/wiki/Time_Stamp_Counter - .. _minimark-environment-variables: Environment variables diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -1,11 +1,12 @@ Historical release notes ======================== -CPython 2.7 compatible versions -------------------------------- +Combined releases +----------------- .. toctree:: + release-v7.1.0.rst release-v7.0.0.rst release-v6.0.0.rst release-v5.10.1.rst @@ -14,6 +15,12 @@ release-v5.8.0.rst release-v5.7.1.rst release-v5.7.0.rst + +CPython 2.7 compatible versions +------------------------------- + +.. toctree:: + release-pypy2.7-v5.6.0.rst release-pypy2.7-v5.4.1.rst release-pypy2.7-v5.4.0.rst @@ -61,15 +68,6 @@ release-0.7.0.rst release-0.6 -CPython 3.5 compatible versions -------------------------------- - -.. toctree:: - - release-v5.8.0.rst - release-v5.7.1.rst - release-v5.7.0.rst - CPython 3.3 compatible versions ------------------------------- diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst --- a/pypy/doc/man/pypy.1.rst +++ b/pypy/doc/man/pypy.1.rst @@ -99,6 +99,8 @@ If set, equivalent to the ``-W`` option (warning control). The value should be a comma-separated list of ``-W`` parameters. +.. _pypylog: + ``PYPYLOG`` If set to a non-empty value, enable logging, the format is: diff --git a/pypy/doc/release-v7.1.0.rst b/pypy/doc/release-v7.1.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v7.1.0.rst @@ -0,0 +1,84 @@ +========================================= +PyPy v7.1.0: release of 2.7, and 3.6-beta +========================================= + +The PyPy team is proud to release the version 7.1.0 of PyPy, which includes +two different interpreters: + + - PyPy2.7, which is an interpreter supporting the syntax and the features of + Python 2.7 + + - PyPy3.6-beta: this is the second official release of PyPy to support 3.6 + features, although it is still considered beta quality. + +The interpreters are based on much the same codebase, thus the double +release. + +This release, coming fast on the heels of 7.0 in February, finally merges the +internal refactoring of unicode representation as UTF-8. Removing the +conversions from strings to unicode internally lead to a nice speed bump. + +We also improved the ability to use the buffer protocol with ctype structures +and arrays. + +Until we can work with downstream providers to distribute builds with PyPy, we +have made packages for some common packages `available as wheels`_. + +As always, this release is 100% compatible with the previous one and fixed +several issues and bugs raised by the growing community of PyPy users. +We strongly recommend updating. + +The PyPy3.6 release is still not production quality so your mileage may vary. +There are open issues with incomplete compatibility and c-extension support. + +You can download the v7.0 releases here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. If PyPy is not quite good enough for your needs, we are available for +direct consulting work. + +We would also like to thank our contributors and encourage new people to join +the project. PyPy has many layers and we need help with all of them: `PyPy`_ +and `RPython`_ documentation improvements, tweaking popular modules to run +on pypy, or general `help`_ with making RPython's JIT even better. + +.. _`PyPy`: index.html +.. _`RPython`: https://rpython.readthedocs.org +.. _`help`: project-ideas.html +.. _`cffi`: http://cffi.readthedocs.io +.. _`cppyy`: https://cppyy.readthedocs.io +.. _`available as wheels`: https://github.com/antocuni/pypy-wheels + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7, 3.6. It's fast (`PyPy and CPython 2.7.x`_ performance +comparison) due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +The PyPy release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +Unfortunately at the moment of writing our ARM buildbots are out of service, +so for now we are **not** releasing any binary for the ARM architecture. + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + + +Changelog +========= + +If not specified, the changes are shared across versions + diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py --- a/pypy/doc/tool/makecontributor.py +++ b/pypy/doc/tool/makecontributor.py @@ -1,4 +1,5 @@ # NOTE: run this script with LANG=en_US.UTF-8 +# works with pip install mercurial==3.0 import py import sys @@ -89,6 +90,7 @@ 'Laurence Tratt': ['ltratt'], 'Pieter Zieschang': ['pzieschang', 'p_zieschang at yahoo.de'], 'John Witulski': ['witulski'], + 'Andrew Lawrence': ['andrew.lawrence at siemens.com', 'andrewjlawrence'], } alias_map = {} diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,34 +1,6 @@ ========================== -What's new in PyPy2.7 7.0+ +What's new in PyPy2.7 7.1+ ========================== -.. this is a revision shortly after release-pypy-7.0.0 -.. startrev: 481c69f7d81f - -.. branch: zlib-copying-third-time-a-charm - -Make sure zlib decompressobjs have their streams deallocated immediately -on flush. - -.. branch: zlib-copying-redux - -Fix calling copy on already-flushed compressobjs. - - - -.. branch: math-improvements - -Improve performance of long operations where one of the operands fits into -an int. - -.. branch: regalloc-playground - -Improve register allocation in the JIT. - -.. branch: promote-unicode - -Implement rlib.jit.promote_unicode to complement promote_string - -.. branch: unicode-utf8 - -Use utf8 internally to represent unicode, with the goal of never using rpython-level unicode +.. this is a revision shortly after release-pypy-7.1.0 +.. startrev: 78914a03cf95 diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-7.1.0.rst copy from pypy/doc/whatsnew-head.rst copy to pypy/doc/whatsnew-pypy2-7.1.0.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-pypy2-7.1.0.rst @@ -32,3 +32,10 @@ .. branch: unicode-utf8 Use utf8 internally to represent unicode, with the goal of never using rpython-level unicode + +.. branch: newmemoryview-app-level + +Since _ctypes is implemented in pure python over libffi, add interfaces and +methods to support the buffer interface from python. Specifically, add a +``__pypy__.newmemoryview`` function to create a memoryview and extend the use +of the PyPy-specific ``__buffer__`` class method. \ No newline at end of file diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-7.1.0.rst copy from pypy/doc/whatsnew-pypy3-head.rst copy to pypy/doc/whatsnew-pypy3-7.1.0.rst diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst --- a/pypy/doc/whatsnew-pypy3-head.rst +++ b/pypy/doc/whatsnew-pypy3-head.rst @@ -1,11 +1,6 @@ ======================== -What's new in PyPy3 7.0+ +What's new in PyPy3 7.1+ ======================== -.. this is the revision after release-pypy3.6-v7.0 -.. startrev: 33fe3b2cf186 - -.. branch: py3.5 - -Merge in py.35 and use this branch as the primary pypy3 one - +.. this is the revision after release-pypy3.6-v7.1 +.. startrev: d642a3c217cb diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -362,6 +362,8 @@ valid so we're trying to either raise or pack stuff with error handler. The key difference is that this is call_may_force """ + if errors is None: + errors = 'strict' slen = len(s) res = StringBuilder(slen) pos = 0 diff --git a/pypy/module/__pypy__/interp_buffer.py b/pypy/module/__pypy__/interp_buffer.py new file mode 100644 --- /dev/null +++ b/pypy/module/__pypy__/interp_buffer.py @@ -0,0 +1,100 @@ +# +# An app-level interface to tp_as_buffer->bf_getbuffer. +# + +from pypy.interpreter.error import oefmt +from pypy.interpreter.gateway import unwrap_spec, interp2app +from pypy.objspace.std.memoryobject import BufferViewND +from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.typedef import TypeDef, generic_new_descr + +class W_Bufferable(W_Root): + def __init__(self, space): + pass + + def descr_buffer(self, space, w_flags): + if type(self) is W_Bufferable: + raise oefmt(space.w_ValueError, "override __buffer__ in a subclass") + return space.call_method(self, '__buffer__', w_flags) + + def readbuf_w(self, space): + mv = space.call_method(self, '__buffer__', space.newint(0)) + return mv.buffer_w(space, 0).as_readbuf() + +W_Bufferable.typedef = TypeDef("Bufferable", None, None, 'read-write', + __doc__ = """a helper class for a app-level class (like _ctypes.Array) +that want to support tp_as_buffer.bf_getbuffer via a __buffer__ method""", + __new__ = generic_new_descr(W_Bufferable), + __buffer__ = interp2app(W_Bufferable.descr_buffer), +) + + at unwrap_spec(itemsize=int, format='text') +def newmemoryview(space, w_obj, itemsize, format, w_shape=None, w_strides=None): + ''' + newmemoryview(buf, itemsize, format, shape=None, strides=None) + ''' + if not space.isinstance_w(w_obj, space.w_memoryview): + raise oefmt(space.w_ValueError, "memoryview expected") + # minimal error checking + lgt = space.len_w(w_obj) + old_size = w_obj.getitemsize() + nbytes = lgt * old_size + if w_shape: + tot = 1 + shape = [] + for w_v in space.listview(w_shape): + v = space.int_w(w_v) + shape.append(v) + tot *= v + if tot * itemsize != nbytes: + raise oefmt(space.w_ValueError, + "shape/itemsize %s/%d does not match obj len/itemsize %d/%d", + str(shape), itemsize, lgt, old_size) + else: + if nbytes % itemsize != 0: + raise oefmt(space.w_ValueError, + "itemsize %d does not match obj len/itemsize %d/%d", + itemsize, lgt, old_size) + shape = [nbytes / itemsize,] + ndim = len(shape) + if w_strides: + strides = [] + for w_v in space.listview(w_strides): + v = space.int_w(w_v) + strides.append(v) + if not w_shape and len(strides) != 1: + raise oefmt(space.w_ValueError, + "strides must have one value if shape not provided") + if len(strides) != ndim: + raise oefmt(space.w_ValueError, + "shape %s does not match strides %s", + str(shape), str(strides)) + else: + # start from the right, c-order layout + strides = [itemsize] * ndim + for v in range(ndim - 2, -1, -1): + strides[v] = strides[v + 1] * shape[v + 1] + # check that the strides are not too big + for i in range(ndim): + if strides[i] * shape[i] > nbytes: + raise oefmt(space.w_ValueError, + "shape %s and strides %s exceed object size %d", + shape, strides, nbytes) + view = space.buffer_w(w_obj, 0) + return space.newmemoryview(FormatBufferViewND(view, itemsize, format, ndim, + shape, strides)) + +class FormatBufferViewND(BufferViewND): + _immutable_ = True + _attrs_ = ['readonly', 'parent', 'ndim', 'shape', 'strides', + 'format', 'itemsize'] + def __init__(self, parent, itemsize, format, ndim, shape, strides): + BufferViewND.__init__(self, parent, ndim, shape, strides) + self.format = format + self.itemsize = itemsize + + def getformat(self): + return self.format + + def getitemsize(self): + return self.itemsize diff --git a/pypy/module/__pypy__/moduledef.py b/pypy/module/__pypy__/moduledef.py --- a/pypy/module/__pypy__/moduledef.py +++ b/pypy/module/__pypy__/moduledef.py @@ -62,11 +62,18 @@ class PyPyDateTime(MixedModule): appleveldefs = {} interpleveldefs = { - 'dateinterop': 'interp_pypydatetime.W_DateTime_Date', - 'timeinterop' : 'interp_pypydatetime.W_DateTime_Time', - 'deltainterop' : 'interp_pypydatetime.W_DateTime_Delta', + 'dateinterop' : 'interp_pypydatetime.W_DateTime_Date', + 'timeinterop' : 'interp_pypydatetime.W_DateTime_Time', + 'deltainterop' : 'interp_pypydatetime.W_DateTime_Delta', } +class PyPyBufferable(MixedModule): + appleveldefs = {} + interpleveldefs = { + 'bufferable': 'interp_buffer.W_Bufferable', + } + + class Module(MixedModule): """ PyPy specific "magic" functions. A lot of them are experimental and subject to change, many are internal. """ @@ -111,6 +118,7 @@ 'fsencode' : 'interp_magic.fsencode', 'fsdecode' : 'interp_magic.fsdecode', 'pyos_inputhook' : 'interp_magic.pyos_inputhook', + 'newmemoryview' : 'interp_buffer.newmemoryview', } submodules = { @@ -120,6 +128,7 @@ "intop": IntOpModule, "os": OsModule, '_pypydatetime': PyPyDateTime, + 'bufferable': PyPyBufferable, } def setup_after_space_initialization(self): diff --git a/pypy/module/__pypy__/test/test_newmemoryview.py b/pypy/module/__pypy__/test/test_newmemoryview.py new file mode 100644 --- /dev/null +++ b/pypy/module/__pypy__/test/test_newmemoryview.py @@ -0,0 +1,32 @@ + + +class AppTestMinimal: + spaceconfig = dict(usemodules=['__pypy__']) + + def test_newmemoryview(self): + from __pypy__ import newmemoryview + b = bytearray(12) + # The format can be anything, we only verify shape, strides, and itemsize + m = newmemoryview(memoryview(b), 2, 'T{= 0 + return max(size, optvarsize) + def write_v(self, cdata, w_ob, optvarsize): # a special case for var-sized C99 arrays from pypy.module._cffi_backend import ctypearray ct = self.ctype + space = ct.space if isinstance(ct, ctypearray.W_CTypeArray) and ct.length < 0: - space = ct.space w_ob, varsizelength = ct.get_new_array_length(w_ob) if optvarsize != -1: # in this mode, the only purpose of this function is to compute # the real size of the structure from a var-sized C99 array assert cdata == lltype.nullptr(rffi.CCHARP.TO) - itemsize = ct.ctitem.size - try: - varsize = ovfcheck(itemsize * varsizelength) - size = ovfcheck(self.offset + varsize) - except OverflowError: - raise oefmt(space.w_OverflowError, - "array size would overflow a ssize_t") - assert size >= 0 - return max(size, optvarsize) + return self.add_varsize_length(space, ct.ctitem.size, + varsizelength, optvarsize) # if 'value' was only an integer, get_new_array_length() returns # w_ob = space.w_None. Detect if this was the case, # and if so, stop here, leaving the content uninitialized @@ -267,6 +273,12 @@ # if optvarsize == -1: self.write(cdata, w_ob) + elif (isinstance(ct, W_CTypeStructOrUnion) and ct._with_var_array and + not isinstance(w_ob, cdataobj.W_CData)): + subsize = ct.size + subsize = ct.convert_struct_from_object( + lltype.nullptr(rffi.CCHARP.TO), w_ob, subsize) + optvarsize = self.add_varsize_length(space, 1, subsize, optvarsize) return optvarsize def convert_bitfield_to_object(self, cdata): diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py --- a/pypy/module/_cffi_backend/newtype.py +++ b/pypy/module/_cffi_backend/newtype.py @@ -368,6 +368,16 @@ raise oefmt(space.w_TypeError, "field '%s.%s' has ctype '%s' of unknown size", w_ctype.name, fname, ftype.name) + elif isinstance(ftype, ctypestruct.W_CTypeStructOrUnion): + ftype.force_lazy_struct() + # GCC (or maybe C99) accepts var-sized struct fields that are not + # the last field of a larger struct. That's why there is no + # check here for "last field": we propagate the flag + # '_with_var_array' to any struct that contains either an open- + # ended array or another struct that recursively contains an + # open-ended array. + if ftype._with_var_array: + with_var_array = True # if is_union: boffset = 0 # reset each field at offset 0 @@ -419,7 +429,6 @@ # a nested anonymous struct or union # note: it seems we only get here with ffi.verify() srcfield2names = {} - ftype.force_lazy_struct() for name, srcfld in ftype._fields_dict.items(): srcfield2names[srcfld] = name for srcfld in ftype._fields_list: diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.12.1", ("This test_c.py file is for testing a version" +assert __version__ == "1.12.2", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): @@ -3441,6 +3441,15 @@ assert p.a[1] == 20 assert p.a[2] == 30 assert p.a[3] == 0 + # + # struct of struct of varsized array + BStruct2 = new_struct_type("bar") + complete_struct_or_union(BStruct2, [('head', BInt), + ('tail', BStruct)]) + for i in range(2): # try to detect heap overwrites + p = newp(new_pointer_type(BStruct2), [100, [200, list(range(50))]]) + assert p.tail.y[49] == 49 + def test_struct_array_no_length_explicit_position(): BInt = new_primitive_type("int") diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -304,7 +304,7 @@ while pos < end: oc = ord(obj[pos]) raw_unicode_escape_helper(builder, oc) - pos += 1 + pos += 1 return space.newtuple([space.newtext(builder.build()), w_end]) else: raise oefmt(space.w_TypeError, @@ -526,7 +526,10 @@ def _call_codec(space, w_coder, w_obj, action, encoding, errors): try: - w_res = space.call_function(w_coder, w_obj, space.newtext(errors)) + if errors: + w_res = space.call_function(w_coder, w_obj, space.newtext(errors)) + else: + w_res = space.call_function(w_coder, w_obj) except OperationError as operr: raise _wrap_codec_error(space, operr, action, encoding) if (not space.isinstance_w(w_res, space.w_tuple) or space.len_w(w_res) != 2): @@ -558,8 +561,8 @@ return w_err_handler - at unwrap_spec(errors='text') -def encode(space, w_obj, w_encoding=None, errors='strict'): + at unwrap_spec(encoding='text_or_none', errors='text_or_none') +def encode(space, w_obj, encoding=None, errors=None): """encode(obj, [encoding[,errors]]) -> object Encodes obj using the codec registered for encoding. encoding defaults @@ -569,20 +572,26 @@ 'xmlcharrefreplace' as well as any other name registered with codecs.register_error that can handle ValueErrors. """ - if w_encoding is None: + if encoding is None: encoding = space.sys.defaultencoding - else: - encoding = space.text_w(w_encoding) w_encoder = space.getitem(lookup_codec(space, encoding), space.newint(0)) - return _call_codec(space, w_encoder, w_obj, "encoding", encoding, errors) + w_retval = _call_codec(space, w_encoder, w_obj, "encoding", encoding, errors) + if not space.isinstance_w(w_retval, space.w_bytes): + raise oefmt(space.w_TypeError, + "'%s' encoder returned '%T' instead of 'bytes'; " + "use codecs.encode() to encode to arbitrary types", + encoding, + w_retval) + return w_retval @unwrap_spec(errors='text_or_none') def readbuffer_encode(space, w_data, errors='strict'): s = space.getarg_w('s#', w_data) return space.newtuple([space.newbytes(s), space.newint(len(s))]) - at unwrap_spec(errors='text') -def decode(space, w_obj, w_encoding=None, errors='strict'): + at unwrap_spec(encoding='text_or_none', errors='text_or_none') +def decode(space, w_obj, encoding=None, errors=None): + from pypy.objspace.std.unicodeobject import W_UnicodeObject """decode(obj, [encoding[,errors]]) -> object Decodes obj using the codec registered for encoding. encoding defaults @@ -592,12 +601,17 @@ as well as any other name registered with codecs.register_error that is able to handle ValueErrors. """ - if w_encoding is None: + if encoding is None: encoding = space.sys.defaultencoding - else: - encoding = space.text_w(w_encoding) w_decoder = space.getitem(lookup_codec(space, encoding), space.newint(1)) - return _call_codec(space, w_decoder, w_obj, "decoding", encoding, errors) + w_retval = _call_codec(space, w_decoder, w_obj, "decoding", encoding, errors) + if not isinstance(w_retval, W_UnicodeObject): + raise oefmt(space.w_TypeError, + "'%s' decoder returned '%T' instead of 'str'; " + "use codecs.decode() to decode to arbitrary types", + encoding, + w_retval) + return w_retval @unwrap_spec(errors='text') def register_error(space, errors, w_handler): @@ -633,20 +647,6 @@ "use %s to handle arbitrary codecs", encoding, action) return codec_info -def encode_text(space, w_obj, encoding, errors): - if errors is None: - errors = 'strict' - w_encoder = space.getitem( - lookup_text_codec(space, "codecs.encode()", encoding), space.newint(0)) - return _call_codec(space, w_encoder, w_obj, "encoding", encoding, errors) - -def decode_text(space, w_obj, encoding, errors): - if errors is None: - errors = 'strict' - w_decoder = space.getitem( - lookup_text_codec(space, "codecs.decode()", encoding), space.newint(1)) - return _call_codec(space, w_decoder, w_obj, "decoding", encoding, errors) - # ____________________________________________________________ def _find_implementation(impl_name): @@ -736,7 +736,7 @@ result = unicodehelper.utf8_encode_utf_8(utf8, errors, state.encode_error_handler, allow_surrogates=False) except unicodehelper.ErrorHandlerError as e: - raise oefmt(space.w_IndexError, + raise oefmt(space.w_IndexError, "position %d from error handler invalid, already encoded %d", e.new,e.old) diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -696,6 +696,22 @@ exc = raises(RuntimeError, u"hello".encode, "test.failingenc") assert exc.value == to_raise + def test_one_arg_encoder(self): + import _codecs + def search_function(encoding): + def encode_one(u): + return (b'foo', len(u)) + def decode_one(u): + return (u'foo', len(u)) + if encoding == 'onearg': + return (encode_one, decode_one, None, None) + return None + _codecs.register(search_function) + assert u"hello".encode("onearg") == b'foo' + assert b"hello".decode("onearg") == u'foo' + assert _codecs.encode(u"hello", "onearg") == b'foo' + assert _codecs.decode(b"hello", "onearg") == u'foo' + def test_cpytest_decode(self): import codecs assert codecs.decode(b'\xe4\xf6\xfc', 'latin-1') == '\xe4\xf6\xfc' diff --git a/pypy/module/_multiprocessing/interp_win32_py3.py b/pypy/module/_multiprocessing/interp_win32_py3.py --- a/pypy/module/_multiprocessing/interp_win32_py3.py +++ b/pypy/module/_multiprocessing/interp_win32_py3.py @@ -9,7 +9,7 @@ message = rwin32.FormatErrorW(errno) w_errcode = space.newint(errno) return OperationError(space.w_WindowsError, - space.newtuple([w_errcode, space.newtext(message), + space.newtuple([w_errcode, space.newtext(*message), space.w_None, w_errcode])) @unwrap_spec(handle=int) diff --git a/pypy/module/_multiprocessing/test/test_semaphore.py b/pypy/module/_multiprocessing/test/test_semaphore.py --- a/pypy/module/_multiprocessing/test/test_semaphore.py +++ b/pypy/module/_multiprocessing/test/test_semaphore.py @@ -11,7 +11,7 @@ 'binascii', 'struct', '_posixsubprocess')) if sys.platform == 'win32': - spaceconfig['usemodules'] += ('_rawffi',) + spaceconfig['usemodules'] += ('_rawffi', '_cffi_backend') else: spaceconfig['usemodules'] += ('fcntl',) diff --git a/pypy/module/_multiprocessing/test/test_win32.py b/pypy/module/_multiprocessing/test/test_win32.py --- a/pypy/module/_multiprocessing/test/test_win32.py +++ b/pypy/module/_multiprocessing/test/test_win32.py @@ -2,7 +2,7 @@ import sys class AppTestWin32: - spaceconfig = dict(usemodules=('_multiprocessing', + spaceconfig = dict(usemodules=('_multiprocessing', '_cffi_backend', 'signal', '_rawffi', 'binascii')) def setup_class(cls): diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py --- a/pypy/module/_rawffi/interp_rawffi.py +++ b/pypy/module/_rawffi/interp_rawffi.py @@ -632,7 +632,7 @@ if _MS_WINDOWS: @unwrap_spec(code=int) def FormatError(space, code): - return space.newtext(rwin32.FormatErrorW(code)) + return space.newtext(*rwin32.FormatErrorW(code)) @unwrap_spec(hresult=int) def check_HRESULT(space, hresult): diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py --- a/pypy/module/_winreg/interp_winreg.py +++ b/pypy/module/_winreg/interp_winreg.py @@ -10,9 +10,9 @@ def raiseWindowsError(space, errcode, context): message = rwin32.FormatErrorW(errcode) w_errcode = space.newint(errcode) - raise OperationError(space.w_WindowsError, - space.newtuple([w_errcode, space.newtext(message), - space.w_None, w_errcode])) + w_t = space.newtuple([w_errcode, space.newtext(*message), + space.w_None, w_errcode]) + raise OperationError(space.w_WindowsError, w_t) class W_HKEY(W_Root): def __init__(self, space, hkey): diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -30,6 +30,7 @@ from pypy.objspace.std.unicodeobject import encode_object from pypy.module.__builtin__.descriptor import W_Property #from pypy.module.micronumpy.base import W_NDimArray +from pypy.module.__pypy__.interp_buffer import W_Bufferable from rpython.rlib.entrypoint import entrypoint_lowlevel from rpython.rlib.rposix import FdValidator from rpython.rlib.unroll import unrolling_iterable @@ -731,6 +732,7 @@ 'PyMethodDescr_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCMethodObject.typedef)', 'PyWrapperDescr_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCWrapperObject.typedef)', 'PyInstanceMethod_Type': 'space.gettypeobject(cpyext.classobject.InstanceMethod.typedef)', + 'PyBufferable_Type': 'space.gettypeobject(W_Bufferable.typedef)', }.items(): register_global(cpyname, 'PyTypeObject*', pypyexpr, header=pypy_decl) diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -32,8 +32,8 @@ * module/sys/version.py * doc/conf.py */ -#define PYPY_VERSION "7.1.0-alpha0" -#define PYPY_VERSION_NUM 0x07010000 +#define PYPY_VERSION "7.2.0-alpha0" +#define PYPY_VERSION_NUM 0x07020000 /* Defined to mean a PyPy where cpyext holds more regular references to PyObjects, e.g. staying alive as long as the internal PyPy object stays alive. */ diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py --- a/pypy/module/cpyext/memoryobject.py +++ b/pypy/module/cpyext/memoryobject.py @@ -43,7 +43,9 @@ fill_Py_buffer(space, w_obj.view, view) try: view.c_buf = rffi.cast(rffi.VOIDP, w_obj.view.get_raw_address()) - view.c_obj = make_ref(space, w_userdata) + # not used in PyPy to keep something alive, + # but some c-extensions check the type without checking for NULL + view.c_obj = make_ref(space, space.w_None) rffi.setintfield(view, 'c_readonly', w_obj.view.readonly) except ValueError: w_s = w_obj.descr_tobytes(space) diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -46,15 +46,15 @@ _dealloc(space, py_obj) def w_kwargs_from_args(space, __args__): - w_kwargs = None - if __args__.keywords: - # CCC: we should probably have a @jit.look_inside_iff if the - # keyword count is constant, as we do in Arguments.unpack - w_kwargs = space.newdict() - for i in range(len(__args__.keywords)): - key = __args__.keywords[i] - w_obj = __args__.keywords_w[i] - space.setitem(w_kwargs, space.newtext(key), w_obj) + if __args__.keywords is None: + return None + # CCC: we should probably have a @jit.look_inside_iff if the + # keyword count is constant, as we do in Arguments.unpack + w_kwargs = space.newdict() + for i in range(len(__args__.keywords)): + key = __args__.keywords[i] + w_obj = __args__.keywords_w[i] + space.setitem(w_kwargs, space.newtext(key), w_obj) return w_kwargs def undotted_name(name): diff --git a/pypy/module/cpyext/parse/cpyext_memoryobject.h b/pypy/module/cpyext/parse/cpyext_memoryobject.h --- a/pypy/module/cpyext/parse/cpyext_memoryobject.h +++ b/pypy/module/cpyext/parse/cpyext_memoryobject.h @@ -1,6 +1,12 @@ /* The struct is declared here but it shouldn't be considered public. Don't access those fields directly, use the functions instead! */ + + +/* this is wrong, PyMemoryViewObject should use PyObject_VAR_HEAD, and use + ob_data[1] to hold the shapes, strides, and offsets for the view. Then + we should use specialized allocators (that break the cpyext model) to + allocate ob_data = malloc(sizeof(Py_ssize_t) * view.ndims * 3) */ typedef struct { PyObject_HEAD Py_buffer view; diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h --- a/pypy/module/cpyext/parse/cpyext_object.h +++ b/pypy/module/cpyext/parse/cpyext_object.h @@ -52,7 +52,8 @@ /* Py3k buffer interface, adapted for PyPy */ -#define Py_MAX_NDIMS 32 +/* XXX remove this constant, us a PyObject_VAR_HEAD instead */ +#define Py_MAX_NDIMS 36 #define Py_MAX_FMT 128 typedef struct bufferinfo { void *buf; diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py --- a/pypy/module/cpyext/test/test_memoryobject.py +++ b/pypy/module/cpyext/test/test_memoryobject.py @@ -36,6 +36,23 @@ decref(space, ref) decref(space, c_memoryview) + def test_class_with___buffer__(self, space, api): + w_obj = space.appexec([], """(): + from __pypy__.bufferable import bufferable + class B(bufferable): + def __init__(self): + self.buf = bytearray(10) + + def __buffer__(self, flags): + return memoryview(self.buf) + return B()""") + py_obj = make_ref(space, w_obj) + assert py_obj.c_ob_type.c_tp_as_buffer + assert py_obj.c_ob_type.c_tp_as_buffer.c_bf_getbuffer + assert py_obj.c_ob_type.c_tp_as_buffer.c_bf_getreadbuffer + assert py_obj.c_ob_type.c_tp_as_buffer.c_bf_getwritebuffer + + class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase): def test_fillWithObject(self): module = self.import_extension('foo', [ diff --git a/pypy/module/cpyext/test/test_methodobject.py b/pypy/module/cpyext/test/test_methodobject.py --- a/pypy/module/cpyext/test/test_methodobject.py +++ b/pypy/module/cpyext/test/test_methodobject.py @@ -87,6 +87,7 @@ assert mod.getarg_KW(a=3, b=4) == ((), {'a': 3, 'b': 4}) assert mod.getarg_KW(1, 2, a=3, b=4) == ((1, 2), {'a': 3, 'b': 4}) assert mod.getarg_KW.__name__ == "getarg_KW" + assert mod.getarg_KW(*(), **{}) == ((), {}) def test_func_attributes(self): diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -13,7 +13,7 @@ # make sure to keep PYPY_VERSION in sync with: # module/cpyext/include/patchlevel.h # doc/conf.py -PYPY_VERSION = (7, 1, 0, "alpha", 0) +PYPY_VERSION = (7, 2, 0, "alpha", 0) import pypy diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py --- a/pypy/objspace/std/stringmethods.py +++ b/pypy/objspace/std/stringmethods.py @@ -193,8 +193,6 @@ from pypy.objspace.std.unicodeobject import ( get_encoding_and_errors, decode_object) encoding, errors = get_encoding_and_errors(space, w_encoding, w_errors) - if errors is None: - errors = 'strict' if encoding is None: encoding = 'utf8' if encoding == 'utf8' or encoding == 'utf-8': diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -41,9 +41,9 @@ if not we_are_translated(): try: # best effort, too expensive to handle surrogates - ulength = len(utf8str.decode('utf8')) + ulength = rutf8.codepoints_in_utf(utf8str) except: - ulength = length + ulength = length assert ulength == length @@ -135,7 +135,7 @@ if strict: raise oefmt(space.w_TypeError, "%s arg must be None, unicode or str", strict) - return unicode_from_encoded_object(space, w_other, 'utf8', "strict") + return decode_object(space, w_other, 'utf8', "strict") def convert_to_w_unicode(self, space): return self @@ -203,8 +203,7 @@ if space.isinstance_w(w_object, space.w_unicode): raise oefmt(space.w_TypeError, "decoding str is not supported") - w_value = unicode_from_encoded_object(space, w_object, - encoding, errors) + w_value = decode_object(space, w_object, encoding, errors) if space.is_w(w_unicodetype, space.w_unicode): return w_value @@ -649,7 +648,7 @@ def descr_startswith(self, space, w_prefix, w_start=None, w_end=None): start, end = self._unwrap_and_compute_idx_params(space, w_start, w_end) value = self._utf8 - if (start > 0 and not space.is_none(w_end) and + if (start > 0 and not space.is_none(w_end) and space.getindex_w(w_end, None) == 0): return space.w_False if space.isinstance_w(w_prefix, space.w_tuple): @@ -674,7 +673,7 @@ start, end = self._unwrap_and_compute_idx_params(space, w_start, w_end) value = self._utf8 # match cpython behaviour - if (start > 0 and not space.is_none(w_end) and + if (start > 0 and not space.is_none(w_end) and space.getindex_w(w_end, None) == 0): return space.w_False if space.isinstance_w(w_suffix, space.w_tuple): @@ -1207,11 +1206,11 @@ errors = None if w_errors is None else space.text_w(w_errors) return encoding, errors - -def encode_object(space, w_object, encoding, errors): - from pypy.module._codecs.interp_codecs import encode_text, CodecState +def encode_object(space, w_obj, encoding, errors): + from pypy.module._codecs.interp_codecs import encode if errors is None or errors == 'strict': - utf8 = space.utf8_w(w_object) + # fast paths + utf8 = space.utf8_w(w_obj) if encoding is None or encoding == 'utf-8': try: rutf8.check_utf8(utf8, False) @@ -1230,22 +1229,12 @@ a.pos, a.pos + 1) assert False, "always raises" return space.newbytes(utf8) - if encoding is None: - encoding = space.sys.defaultencoding - w_retval = encode_text(space, w_object, encoding, errors) - if not space.isinstance_w(w_retval, space.w_bytes): - raise oefmt(space.w_TypeError, - "'%s' encoder returned '%T' instead of 'bytes'; " - "use codecs.encode() to encode to arbitrary types", - encoding, - w_retval) - return w_retval + return encode(space, w_obj, encoding, errors) -def decode_object(space, w_obj, encoding, errors='strict'): - assert errors is not None - assert encoding is not None - if errors == 'strict': +def decode_object(space, w_obj, encoding, errors=None): + if errors == 'strict' or errors is None: + # fast paths if encoding == 'ascii': s = space.charbuf_w(w_obj) unicodehelper.check_ascii_or_raise(space, s) @@ -1254,29 +1243,8 @@ s = space.charbuf_w(w_obj) lgt = unicodehelper.check_utf8_or_raise(space, s) return space.newutf8(s, lgt) - from pypy.module._codecs.interp_codecs import decode_text - w_retval = decode_text(space, w_obj, encoding, errors) - if not isinstance(w_retval, W_UnicodeObject): - raise oefmt(space.w_TypeError, - "'%s' decoder returned '%T' instead of 'str'; " - "use codecs.decode() to decode to arbitrary types", - encoding, - w_retval) - return w_retval - - -def unicode_from_encoded_object(space, w_obj, encoding, errors): - if errors is None: - errors = 'strict' - if encoding is None: - encoding = getdefaultencoding(space) - w_retval = decode_object(space, w_obj, encoding, errors) - if not isinstance(w_retval, W_UnicodeObject): - raise oefmt(space.w_TypeError, - "decoder did not return a str object (type '%T')", - w_retval) - return w_retval - + from pypy.module._codecs.interp_codecs import decode + return decode(space, w_obj, encoding, errors) def unicode_from_object(space, w_obj): if space.is_w(space.type(w_obj), space.w_unicode): @@ -1284,6 +1252,7 @@ if space.lookup(w_obj, "__str__") is not None: return space.str(w_obj) return space.repr(w_obj) + def ascii_from_object(space, w_obj): """Implements builtins.ascii()""" # repr is guaranteed to be unicode @@ -1295,7 +1264,7 @@ # this is a performance and bootstrapping hack encoding = getdefaultencoding(space) if encoding != 'ascii': - return unicode_from_encoded_object(space, w_bytes, encoding, "strict") + return decode_object(space, w_bytes, encoding, "strict") s = space.bytes_w(w_bytes) unicodehelper.check_ascii_or_raise(space, s) return W_UnicodeObject(s, len(s)) @@ -1900,7 +1869,7 @@ if not isinstance(w_unistr, W_UnicodeObject): raise oefmt(space.w_TypeError, "expected unicode, got '%T'", w_unistr) utf8 = space.utf8_w(w_unistr) - lgt = space.len_w(w_unistr) + lgt = space.len_w(w_unistr) result = StringBuilder(lgt) pos = 0 for uchr in rutf8.Utf8StringIterator(utf8): @@ -1923,7 +1892,7 @@ raise OperationError(space.w_UnicodeEncodeError, space.newtuple([w_encoding, w_unistr, w_start, w_end, - w_reason])) + w_reason])) result.append(c) pos += 1 return result.build() From pypy.commits at gmail.com Tue Mar 12 15:14:10 2019 From: pypy.commits at gmail.com (rlamy) Date: Tue, 12 Mar 2019 12:14:10 -0700 (PDT) Subject: [pypy-commit] pypy py3tests: dummy merge with apptest-file Message-ID: <5c880502.1c69fb81.d3446.3ab3@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r96300:39bc5e00c7a8 Date: 2019-03-12 19:11 +0000 http://bitbucket.org/pypy/pypy/changeset/39bc5e00c7a8/ Log: dummy merge with apptest-file From pypy.commits at gmail.com Tue Mar 12 19:04:19 2019 From: pypy.commits at gmail.com (andrewjlawrence) Date: Tue, 12 Mar 2019 16:04:19 -0700 (PDT) Subject: [pypy-commit] pypy winoverlapped: Fixed ProactorTests.test_pipe Message-ID: <5c883af3.1c69fb81.3afb0.016f@mx.google.com> Author: andrewjlawrence Branch: winoverlapped Changeset: r96303:9962c2550731 Date: 2019-03-12 23:02 +0000 http://bitbucket.org/pypy/pypy/changeset/9962c2550731/ Log: Fixed ProactorTests.test_pipe diff --git a/lib_pypy/_overlapped.py b/lib_pypy/_overlapped.py --- a/lib_pypy/_overlapped.py +++ b/lib_pypy/_overlapped.py @@ -64,8 +64,8 @@ def __init__(self, event): self.overlapped = _ffi.new('OVERLAPPED[1]') self.handle = _ffi.NULL - self.allocated_buffer = None - self.user_buffer = None + self.read_buffer = None + self.write_buffer = None self.type = OverlappedType.TYPE_NONE if event == INVALID_HANDLE_VALUE or not event: @@ -121,7 +121,7 @@ raise _winapi.WinError() if self.type == OverlappedType.TYPE_READ: - return _ffi.unpack(self.allocated_buffer, transferred[0]) + return _ffi.unpack(self.read_buffer, transferred[0]) else: return transferred[0] @@ -147,8 +147,8 @@ self.type = OverlappedType.TYPE_READ self.handle = _int2handle(handle) - self.allocated_buffer = _ffi.new("CHAR[]", max(1,size)) - return self.do_WSARecv(handle, self.allocated_buffer, size, flags) + self.read_buffer = _ffi.new("CHAR[]", max(1,size)) + return self.do_WSARecv(handle, self.read_buffer, size, flags) def do_WSARecv(self, handle, allocatedbuffer, size, flags): nread = _ffi.new("LPDWORD") @@ -201,8 +201,8 @@ def ReadFile(self, handle, size): self.type = OverlappedType.TYPE_READ self.handle = _int2handle(handle) - self.allocated_buffer = _ffi.new("CHAR[]", max(1,size)) - return self.do_ReadFile(self.handle, self.allocated_buffer, size) + self.read_buffer = _ffi.new("CHAR[]", max(1,size)) + return self.do_ReadFile(self.handle, self.read_buffer, size) def do_ReadFile(self, handle, buf, size): nread = _ffi.new('DWORD[1]', [0]) @@ -220,6 +220,32 @@ else: self.type = OverlappedType.TYPE_NOT_STARTED raise _winapi._WinError() + + def WriteFile(self, handle, buffer): + self.handle = _int2handle(handle) + self.write_buffer = buffer + written = _ffi.new('DWORD[1]', [0]) + + # Check if we have already performed some IO + if self.type != OverlappedType.TYPE_NONE: + raise _winapi._WinError() + + self.type = OverlappedType.TYPE_WRITE + + ret = _kernel32.WriteFile(self.handle, self.write_buffer, len(self.write_buffer), written, self.overlapped) + + if ret: + err = _winapi.ERROR_SUCCESS + else: + err = _winapi.GetLastError() + + self.error = err + + if err == _winapi.ERROR_SUCCESS or err == _winapi.ERROR_IO_PENDING: + return None + else: + self.type = OverlappedType.TYPE_NOT_STARTED + raise _winapi.WinError() @property def pending(self): @@ -326,6 +352,14 @@ return _handle2int(handle) +#def UnregisterWaitEx(handle, event): +# waithandle = _int2handle(handle) +# waitevent = _int2handle(event) +# +# ret = _kernel32.UnregisterWaitEx(handle, event) +# +# if not ret: +# raise _winapi.WinError() # In CPython this function converts a windows error into a python object # Not sure what we should do here. diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py --- a/lib_pypy/_pypy_winbase_build.py +++ b/lib_pypy/_pypy_winbase_build.py @@ -139,10 +139,13 @@ typedef VOID (WINAPI *WAITORTIMERCALLBACK) (PVOID, BOOL); BOOL WINAPI RegisterWaitForSingleObject(PHANDLE, HANDLE, WAITORTIMERCALLBACK, PVOID, ULONG, ULONG); BOOL WINAPI PostQueuedCompletionStatus(HANDLE, DWORD, ULONG_PTR, LPOVERLAPPED); +BOOL WINAPI UnregisterWaitEx(HANDLE, HANDLE); BOOL WINAPI GetQueuedCompletionStatus(HANDLE, LPDWORD, ULONG**, LPOVERLAPPED*, DWORD); HANDLE WINAPI CreateIoCompletionPort(HANDLE, HANDLE, ULONG_PTR, DWORD); +BOOL WINAPI WriteFile(HANDLE, LPCVOID, DWORD, LPDWORD, LPOVERLAPPED); + #define WT_EXECUTEINWAITTHREAD 0x00000004 #define WT_EXECUTEONLYONCE 0x00000008 diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py --- a/lib_pypy/_pypy_winbase_cffi.py +++ b/lib_pypy/_pypy_winbase_cffi.py @@ -3,8 +3,8 @@ ffi = _cffi_backend.FFI('_pypy_winbase_cffi', _version = 0x2601, - _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\xF0\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x13\x03\x00\x00\xE3\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\xEA\x03\x00\x00\x1A\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x1A\x11\x00\x00\x1A\x11\x00\x00\xE8\x03\x00\x00\xE4\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x03\x00\x00\x26\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x26\x11\x00\x00\x11\x11\x00\x00\xDF\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x14\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\xE9\x03\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\xDA\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x14\x03\x00\x00\x15\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x26\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x96\x03\x00\x00\x74\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x74\x11\x00\x00\x74\x11\x00\x00\x22\x11\x00\x00\x23\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x51\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x11\x11\x00\x00\x74\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x02\x0F\x00\x00\x91\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x91\x0D\x00\x00\x00\x0F\x00\x00\x91\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x00\xE7\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xEA\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x99\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x96\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x9C\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x99\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x9C\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x99\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x74\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x99\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xA2\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x99\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\xF0\x0D\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x00\x0F\x00\x00\xF0\x0D\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x04\x09\x00\x00\x02\x09\x00\x00\xE6\x03\x00\x00\x05\x09\x00\x00\x06\x09\x00\x00\x03\x09\x00\x00\x07\x09\x00\x00\x02\x01\x00\x00\x40\x03\x00\x00\x01\x09\x00\x00\x00\x09\x00\x00\xEF\x03\x00\x00\x04\x01\x00\x00\x00\x01', - _globals = (b'\x00\x00\x33\x23CancelIo',0,b'\x00\x00\x36\x23CancelIoEx',0,b'\x00\x00\x33\x23CloseHandle',0,b'\x00\x00\x36\x23ConnectNamedPipe',0,b'\x00\x00\x98\x23CreateEventA',0,b'\x00\x00\x9E\x23CreateEventW',0,b'\x00\x00\xA4\x23CreateFileA',0,b'\x00\x00\xD1\x23CreateFileW',0,b'\x00\x00\xBF\x23CreateIoCompletionPort',0,b'\x00\x00\xAD\x23CreateNamedPipeA',0,b'\x00\x00\xC7\x23CreateNamedPipeW',0,b'\x00\x00\x25\x23CreatePipe',0,b'\x00\x00\x19\x23CreateProcessA',0,b'\x00\x00\x73\x23CreateProcessW',0,b'\x00\x00\x6A\x23DuplicateHandle',0,b'\x00\x00\xC5\x23GetCurrentProcess',0,b'\x00\x00\x53\x23GetExitCodeProcess',0,b'\x00\x00\x8E\x23GetLastError',0,b'\x00\x00\x89\x23GetModuleFileNameW',0,b'\x00\x00\x3A\x23GetOverlappedResult',0,b'\x00\x00\x57\x23GetQueuedCompletionStatus',0,b'\x00\x00\xBC\x23GetStdHandle',0,b'\x00\x00\x8E\x23GetVersion',0,b'\x00\x00\x64\x23PostQueuedCompletionStatus',0,b'\x00\x00\x10\x23ReadFile',0,b'\x00\x00\x2B\x23RegisterWaitForSingleObject',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\x82\x23SetErrorMode',0,b'\x00\x00\x33\x23SetEvent',0,b'\x00\x00\x5E\x23SetNamedPipeHandleState',0,b'\x00\x00\x4F\x23TerminateProcess',0,b'\x00\x00\x46\x23WSARecv',0,b'\xFF\xFF\xFF\x1FWT_EXECUTEINWAITTHREAD',4,b'\xFF\xFF\xFF\x1FWT_EXECUTEONLYONCE',8,b'\x00\x00\x85\x23WaitForSingleObject',0,b'\x00\x00\x7F\x23_get_osfhandle',0,b'\x00\x00\x17\x23_getch',0,b'\x00\x00\x17\x23_getche',0,b'\x00\x00\x93\x23_getwch',0,b'\x00\x00\x93\x23_getwche',0,b'\x00\x00\x17\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x95\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x90\x23_ungetwch',0,b'\x00\x00\xB7\x23socket',0), - _struct_unions = ((b'\x00\x00\x00\xED\x00\x00\x00\x03$1',b'\x00\x00\xEC\x11DUMMYSTRUCTNAME',b'\x00\x00\x11\x11Pointer'),(b'\x00\x00\x00\xEC\x00\x00\x00\x02$2',b'\x00\x00\x13\x11Offset',b'\x00\x00\x13\x11OffsetHigh'),(b'\x00\x00\x00\xE4\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x11\x11hProcess',b'\x00\x00\x11\x11hThread',b'\x00\x00\x13\x11dwProcessId',b'\x00\x00\x13\x11dwThreadId'),(b'\x00\x00\x00\xE8\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x13\x11cb',b'\x00\x00\x1A\x11lpReserved',b'\x00\x00\x1A\x11lpDesktop',b'\x00\x00\x1A\x11lpTitle',b'\x00\x00\x13\x11dwX',b'\x00\x00\x13\x11dwY',b'\x00\x00\x13\x11dwXSize',b'\x00\x00\x13\x11dwYSize',b'\x00\x00\x13\x11dwXCountChars',b'\x00\x00\x13\x11dwYCountChars',b'\x00\x00\x13\x11dwFillAttribute',b'\x00\x00\x13\x11dwFlags',b'\x00\x00\x91\x11wShowWindow',b'\x00\x00\x91\x11cbReserved2',b'\x00\x00\xEE\x11lpReserved2',b'\x00\x00\x11\x11hStdInput',b'\x00\x00\x11\x11hStdOutput',b'\x00\x00\x11\x11hStdError'),(b'\x00\x00\x00\xE3\x00\x00\x00\x02_OVERLAPPED',b'\x00\x00\x13\x11Internal',b'\x00\x00\x13\x11InternalHigh',b'\x00\x00\xED\x11DUMMYUNIONNAME',b'\x00\x00\x11\x11hEvent'),(b'\x00\x00\x00\xE6\x00\x00\x00\x02_PostCallbackData',b'\x00\x00\x11\x11hCompletionPort',b'\x00\x00\x15\x11Overlapped'),(b'\x00\x00\x00\xE7\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x13\x11nLength',b'\x00\x00\x11\x11lpSecurityDescriptor',b'\x00\x00\x01\x11bInheritHandle'),(b'\x00\x00\x00\xE9\x00\x00\x00\x02_WSABUF',b'\x00\x00\x13\x11len',b'\x00\x00\x1A\x11buf')), - _typenames = (b'\x00\x00\x00\xEBLPFN_DISCONNECTEX',b'\x00\x00\x00\x15LPOVERLAPPED',b'\x00\x00\x00\x4DLPOVERLAPPED_COMPLETION_ROUTINE',b'\x00\x00\x00\x23LPPROCESS_INFORMATION',b'\x00\x00\x00\xE5LPPostCallbackData',b'\x00\x00\x00\x99LPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x22LPSTARTUPINFO',b'\x00\x00\x00\x48LPWSABUF',b'\x00\x00\x00\xE3OVERLAPPED',b'\x00\x00\x00\xE4PROCESS_INFORMATION',b'\x00\x00\x00\x99PSECURITY_ATTRIBUTES',b'\x00\x00\x00\xE6PostCallbackData',b'\x00\x00\x00\xE7SECURITY_ATTRIBUTES',b'\x00\x00\x00\x11SOCKET',b'\x00\x00\x00\xE8STARTUPINFO',b'\x00\x00\x00\x2EWAITORTIMERCALLBACK',b'\x00\x00\x00\xE9WSABUF',b'\x00\x00\x00\x91wint_t'), + _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\xFB\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x13\x03\x00\x00\xEE\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\xF5\x03\x00\x00\x1A\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x1A\x11\x00\x00\x1A\x11\x00\x00\xF3\x03\x00\x00\xEF\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x03\x00\x00\x26\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x26\x11\x00\x00\x11\x11\x00\x00\xEA\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x14\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\xF4\x03\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\xE5\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x14\x03\x00\x00\x15\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x26\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\xFB\x03\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\xA1\x03\x00\x00\x7F\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x7F\x11\x00\x00\x7F\x11\x00\x00\x22\x11\x00\x00\x23\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x51\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x11\x11\x00\x00\x7F\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x02\x0F\x00\x00\x9C\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x9C\x0D\x00\x00\x00\x0F\x00\x00\x9C\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x00\xF2\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xF5\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xA4\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xA1\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xA7\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xA4\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xA7\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xA4\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x7F\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xA4\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xAD\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xA4\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\xFB\x0D\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x00\x0F\x00\x00\xFB\x0D\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x04\x09\x00\x00\x02\x09\x00\x00\xF1\x03\x00\x00\x05\x09\x00\x00\x06\x09\x00\x00\x03\x09\x00\x00\x07\x09\x00\x00\x02\x01\x00\x00\x40\x03\x00\x00\x01\x09\x00\x00\x00\x09\x00\x00\xFA\x03\x00\x00\x04\x01\x00\x00\x00\x01', + _globals = (b'\x00\x00\x33\x23CancelIo',0,b'\x00\x00\x36\x23CancelIoEx',0,b'\x00\x00\x33\x23CloseHandle',0,b'\x00\x00\x36\x23ConnectNamedPipe',0,b'\x00\x00\xA3\x23CreateEventA',0,b'\x00\x00\xA9\x23CreateEventW',0,b'\x00\x00\xAF\x23CreateFileA',0,b'\x00\x00\xDC\x23CreateFileW',0,b'\x00\x00\xCA\x23CreateIoCompletionPort',0,b'\x00\x00\xB8\x23CreateNamedPipeA',0,b'\x00\x00\xD2\x23CreateNamedPipeW',0,b'\x00\x00\x25\x23CreatePipe',0,b'\x00\x00\x19\x23CreateProcessA',0,b'\x00\x00\x7E\x23CreateProcessW',0,b'\x00\x00\x6E\x23DuplicateHandle',0,b'\x00\x00\xD0\x23GetCurrentProcess',0,b'\x00\x00\x53\x23GetExitCodeProcess',0,b'\x00\x00\x99\x23GetLastError',0,b'\x00\x00\x94\x23GetModuleFileNameW',0,b'\x00\x00\x3A\x23GetOverlappedResult',0,b'\x00\x00\x57\x23GetQueuedCompletionStatus',0,b'\x00\x00\xC7\x23GetStdHandle',0,b'\x00\x00\x99\x23GetVersion',0,b'\x00\x00\x64\x23PostQueuedCompletionStatus',0,b'\x00\x00\x10\x23ReadFile',0,b'\x00\x00\x2B\x23RegisterWaitForSingleObject',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\x8D\x23SetErrorMode',0,b'\x00\x00\x33\x23SetEvent',0,b'\x00\x00\x5E\x23SetNamedPipeHandleState',0,b'\x00\x00\x4F\x23TerminateProcess',0,b'\x00\x00\x6A\x23UnregisterWaitEx',0,b'\x00\x00\x46\x23WSARecv',0,b'\xFF\xFF\xFF\x1FWT_EXECUTEINWAITTHREAD',4,b'\xFF\xFF\xFF\x1FWT_EXECUTEONLYONCE',8,b'\x00\x00\x90\x23WaitForSingleObject',0,b'\x00\x00\x77\x23WriteFile',0,b'\x00\x00\x8A\x23_get_osfhandle',0,b'\x00\x00\x17\x23_getch',0,b'\x00\x00\x17\x23_getche',0,b'\x00\x00\x9E\x23_getwch',0,b'\x00\x00\x9E\x23_getwche',0,b'\x00\x00\x17\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\xA0\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x9B\x23_ungetwch',0,b'\x00\x00\xC2\x23socket',0), + _struct_unions = ((b'\x00\x00\x00\xF8\x00\x00\x00\x03$1',b'\x00\x00\xF7\x11DUMMYSTRUCTNAME',b'\x00\x00\x11\x11Pointer'),(b'\x00\x00\x00\xF7\x00\x00\x00\x02$2',b'\x00\x00\x13\x11Offset',b'\x00\x00\x13\x11OffsetHigh'),(b'\x00\x00\x00\xEF\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x11\x11hProcess',b'\x00\x00\x11\x11hThread',b'\x00\x00\x13\x11dwProcessId',b'\x00\x00\x13\x11dwThreadId'),(b'\x00\x00\x00\xF3\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x13\x11cb',b'\x00\x00\x1A\x11lpReserved',b'\x00\x00\x1A\x11lpDesktop',b'\x00\x00\x1A\x11lpTitle',b'\x00\x00\x13\x11dwX',b'\x00\x00\x13\x11dwY',b'\x00\x00\x13\x11dwXSize',b'\x00\x00\x13\x11dwYSize',b'\x00\x00\x13\x11dwXCountChars',b'\x00\x00\x13\x11dwYCountChars',b'\x00\x00\x13\x11dwFillAttribute',b'\x00\x00\x13\x11dwFlags',b'\x00\x00\x9C\x11wShowWindow',b'\x00\x00\x9C\x11cbReserved2',b'\x00\x00\xF9\x11lpReserved2',b'\x00\x00\x11\x11hStdInput',b'\x00\x00\x11\x11hStdOutput',b'\x00\x00\x11\x11hStdError'),(b'\x00\x00\x00\xEE\x00\x00\x00\x02_OVERLAPPED',b'\x00\x00\x13\x11Internal',b'\x00\x00\x13\x11InternalHigh',b'\x00\x00\xF8\x11DUMMYUNIONNAME',b'\x00\x00\x11\x11hEvent'),(b'\x00\x00\x00\xF1\x00\x00\x00\x02_PostCallbackData',b'\x00\x00\x11\x11hCompletionPort',b'\x00\x00\x15\x11Overlapped'),(b'\x00\x00\x00\xF2\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x13\x11nLength',b'\x00\x00\x11\x11lpSecurityDescriptor',b'\x00\x00\x01\x11bInheritHandle'),(b'\x00\x00\x00\xF4\x00\x00\x00\x02_WSABUF',b'\x00\x00\x13\x11len',b'\x00\x00\x1A\x11buf')), + _typenames = (b'\x00\x00\x00\xF6LPFN_DISCONNECTEX',b'\x00\x00\x00\x15LPOVERLAPPED',b'\x00\x00\x00\x4DLPOVERLAPPED_COMPLETION_ROUTINE',b'\x00\x00\x00\x23LPPROCESS_INFORMATION',b'\x00\x00\x00\xF0LPPostCallbackData',b'\x00\x00\x00\xA4LPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x22LPSTARTUPINFO',b'\x00\x00\x00\x48LPWSABUF',b'\x00\x00\x00\xEEOVERLAPPED',b'\x00\x00\x00\xEFPROCESS_INFORMATION',b'\x00\x00\x00\xA4PSECURITY_ATTRIBUTES',b'\x00\x00\x00\xF1PostCallbackData',b'\x00\x00\x00\xF2SECURITY_ATTRIBUTES',b'\x00\x00\x00\x11SOCKET',b'\x00\x00\x00\xF3STARTUPINFO',b'\x00\x00\x00\x2EWAITORTIMERCALLBACK',b'\x00\x00\x00\xF4WSABUF',b'\x00\x00\x00\x9Cwint_t'), ) diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py --- a/lib_pypy/_winapi.py +++ b/lib_pypy/_winapi.py @@ -18,14 +18,10 @@ # Now the _subprocess module implementation -PermissionErrors = [5] def _WinError(): code, message = _ffi.getwinerror() - if code in PermissionErrors: - excep = PermissionError(code, message) - else: - excep = WindowsError(code, message) + excep = WindowsError(None, message, None ,code) raise excep def _int2handle(val): From pypy.commits at gmail.com Wed Mar 13 01:26:52 2019 From: pypy.commits at gmail.com (mattip) Date: Tue, 12 Mar 2019 22:26:52 -0700 (PDT) Subject: [pypy-commit] pypy default: avoid using range Message-ID: <5c88949c.1c69fb81.570f5.458d@mx.google.com> Author: Matti Picus Branch: Changeset: r96304:ab0f20d4a572 Date: 2019-03-13 07:24 +0200 http://bitbucket.org/pypy/pypy/changeset/ab0f20d4a572/ Log: avoid using range diff --git a/rpython/rlib/rsre/rsre_utf8.py b/rpython/rlib/rsre/rsre_utf8.py --- a/rpython/rlib/rsre/rsre_utf8.py +++ b/rpython/rlib/rsre/rsre_utf8.py @@ -40,17 +40,23 @@ prev_indirect = prev def next_n(self, position, n, end_position): - for i in range(n): + i = 0 + # avoid range(n) since n can be quite large + while i < n: if position >= end_position: raise EndOfString position = rutf8.next_codepoint_pos(self._utf8, position) + i += 1 return position def prev_n(self, position, n, start_position): - for i in range(n): + i = 0 + # avoid range(n) since n can be quite large + while i < n: if position <= start_position: raise EndOfString position = rutf8.prev_codepoint_pos(self._utf8, position) + i += 1 assert position >= 0 return position From pypy.commits at gmail.com Wed Mar 13 01:26:54 2019 From: pypy.commits at gmail.com (mattip) Date: Tue, 12 Mar 2019 22:26:54 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: merge default into branch Message-ID: <5c88949e.1c69fb81.fa0f9.2fc6@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r96305:2158401a5481 Date: 2019-03-13 07:26 +0200 http://bitbucket.org/pypy/pypy/changeset/2158401a5481/ Log: merge default into branch diff --git a/rpython/rlib/rsre/rsre_utf8.py b/rpython/rlib/rsre/rsre_utf8.py --- a/rpython/rlib/rsre/rsre_utf8.py +++ b/rpython/rlib/rsre/rsre_utf8.py @@ -40,17 +40,23 @@ prev_indirect = prev def next_n(self, position, n, end_position): - for i in range(n): + i = 0 + # avoid range(n) since n can be quite large + while i < n: if position >= end_position: raise EndOfString position = rutf8.next_codepoint_pos(self._utf8, position) + i += 1 return position def prev_n(self, position, n, start_position): - for i in range(n): + i = 0 + # avoid range(n) since n can be quite large + while i < n: if position <= start_position: raise EndOfString position = rutf8.prev_codepoint_pos(self._utf8, position) + i += 1 assert position >= 0 return position From pypy.commits at gmail.com Wed Mar 13 15:05:00 2019 From: pypy.commits at gmail.com (rlamy) Date: Wed, 13 Mar 2019 12:05:00 -0700 (PDT) Subject: [pypy-commit] pypy py3tests: Fix OrderedDict pickling Message-ID: <5c89545c.1c69fb81.dd8e4.5c10@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r96306:749113048c38 Date: 2019-03-13 19:04 +0000 http://bitbucket.org/pypy/pypy/changeset/749113048c38/ Log: Fix OrderedDict pickling diff --git a/pypy/module/_collections/app_odict.py b/pypy/module/_collections/app_odict.py --- a/pypy/module/_collections/app_odict.py +++ b/pypy/module/_collections/app_odict.py @@ -14,6 +14,8 @@ cases but is nonsensical in other cases. This is officially forbidden by the CPython docs, so we forbid it explicitly for now. ''' + __module__ = 'collections' + def __init__(*args, **kwds): '''Initialize an ordered dictionary. The signature is the same as regular dictionaries, but keyword arguments are not recommended because From pypy.commits at gmail.com Wed Mar 13 17:11:44 2019 From: pypy.commits at gmail.com (rlamy) Date: Wed, 13 Mar 2019 14:11:44 -0700 (PDT) Subject: [pypy-commit] pypy apptest-file: Fix MixedModule.get_applevel_name() Message-ID: <5c897210.1c69fb81.58c81.f6bf@mx.google.com> Author: Ronan Lamy Branch: apptest-file Changeset: r96307:954d1a568c61 Date: 2019-03-13 21:11 +0000 http://bitbucket.org/pypy/pypy/changeset/954d1a568c61/ Log: Fix MixedModule.get_applevel_name() diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py --- a/pypy/interpreter/mixedmodule.py +++ b/pypy/interpreter/mixedmodule.py @@ -74,8 +74,7 @@ if cls.applevel_name is not None: return cls.applevel_name else: - pkgroot = cls.__module__ - return pkgroot.split('.')[-1] + return cls.__module__.split('.')[-2] def get(self, name): space = self.space diff --git a/pypy/interpreter/test/test_appinterp.py b/pypy/interpreter/test/test_appinterp.py --- a/pypy/interpreter/test/test_appinterp.py +++ b/pypy/interpreter/test/test_appinterp.py @@ -129,6 +129,7 @@ assert name in module.__dict__ """) assert space.is_true(w_module.call('somefunc')) + assert Module.get_applevel_name() == 'demomixedmod' def test_whacking_at_loaders(self): """Some MixedModules change 'self.loaders' in __init__(), but doing From pypy.commits at gmail.com Wed Mar 13 17:14:45 2019 From: pypy.commits at gmail.com (rlamy) Date: Wed, 13 Mar 2019 14:14:45 -0700 (PDT) Subject: [pypy-commit] pypy py3tests: hg merge apptest-file Message-ID: <5c8972c5.1c69fb81.10d2f.9dc3@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r96308:243d7e29848d Date: 2019-03-13 21:14 +0000 http://bitbucket.org/pypy/pypy/changeset/243d7e29848d/ Log: hg merge apptest-file diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py --- a/pypy/interpreter/mixedmodule.py +++ b/pypy/interpreter/mixedmodule.py @@ -79,8 +79,7 @@ if cls.applevel_name is not None: return cls.applevel_name else: - pkgroot = cls.__module__ - return pkgroot.split('.')[-1] + return cls.__module__.split('.')[-2] def get(self, name): space = self.space diff --git a/pypy/interpreter/test/test_appinterp.py b/pypy/interpreter/test/test_appinterp.py --- a/pypy/interpreter/test/test_appinterp.py +++ b/pypy/interpreter/test/test_appinterp.py @@ -129,6 +129,7 @@ assert name in module.__dict__ """) assert space.is_true(w_module.call('somefunc')) + assert Module.get_applevel_name() == 'demomixedmod' def test_whacking_at_loaders(self): """Some MixedModules change 'self.loaders' in __init__(), but doing From pypy.commits at gmail.com Thu Mar 14 01:53:26 2019 From: pypy.commits at gmail.com (mattip) Date: Wed, 13 Mar 2019 22:53:26 -0700 (PDT) Subject: [pypy-commit] pypy default: average_size was a no-op and has been removed Message-ID: <5c89ec56.1c69fb81.fa0e4.54a2@mx.google.com> Author: Matti Picus Branch: Changeset: r96309:844d3753b3a9 Date: 2019-03-14 07:30 +0200 http://bitbucket.org/pypy/pypy/changeset/844d3753b3a9/ Log: average_size was a no-op and has been removed https://hypothesis.readthedocs.io/en/latest/changes.html#strategies diff --git a/rpython/rlib/test/test_rawrefcount_boehm.py b/rpython/rlib/test/test_rawrefcount_boehm.py --- a/rpython/rlib/test/test_rawrefcount_boehm.py +++ b/rpython/rlib/test/test_rawrefcount_boehm.py @@ -111,7 +111,7 @@ pyobjs.append(varname) return varname - for op in draw(strategies.lists(operations, average_size=250)): + for op in draw(strategies.lists(operations)): if op == 'new_gcobj': new_gcobj() elif op == 'new_pyobj': From pypy.commits at gmail.com Thu Mar 14 11:18:55 2019 From: pypy.commits at gmail.com (mattip) Date: Thu, 14 Mar 2019 08:18:55 -0700 (PDT) Subject: [pypy-commit] pypy issue2968: rework test for python2, python3 Message-ID: <5c8a70df.1c69fb81.64ec5.e9a0@mx.google.com> Author: Matti Picus Branch: issue2968 Changeset: r96310:4285c5076c8e Date: 2019-03-14 17:03 +0200 http://bitbucket.org/pypy/pypy/changeset/4285c5076c8e/ Log: rework test for python2, python3 diff --git a/pypy/module/cpyext/test/THPSize.c b/pypy/module/cpyext/test/THPSize.c deleted file mode 100644 --- a/pypy/module/cpyext/test/THPSize.c +++ /dev/null @@ -1,85 +0,0 @@ -#include "Python.h" - -struct THPSize { - PyTupleObject tuple; -} THPSize; - -static PyObject * THPSize_pynew(PyTypeObject *type, PyObject *args, PyObject *kwargs) -{ - return PyTuple_Type.tp_new(type, args, kwargs); -} - -static PyMappingMethods THPSize_as_mapping = { - 0, //PyTuple_Type.tp_as_mapping->mp_length, - 0, - 0 -}; - - -PyTypeObject THPSizeType = { - PyVarObject_HEAD_INIT(0, 0) - "torch.Size", /* tp_name */ - sizeof(THPSize), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - &THPSize_as_mapping, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - &PyTuple_Type, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - THPSize_pynew, /* tp_new */ -}; - -static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - "THPSize", - "Module Doc", - -1, - NULL, - NULL, - NULL, - NULL, - NULL, -}; - -PyMODINIT_FUNC -PyInit_THPSize(void) -{ - PyObject *module = PyModule_Create(&moduledef); - THPSize_as_mapping.mp_length = PyTuple_Type.tp_as_mapping->mp_length; - if (PyType_Ready(&THPSizeType) < 0) { - return NULL; - } - Py_INCREF(&THPSizeType); - if (PyModule_AddObject(module, "Size", (PyObject*)&THPSizeType) < 0) { - return NULL; - } - return module; -} diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py --- a/pypy/module/cpyext/test/test_tupleobject.py +++ b/pypy/module/cpyext/test/test_tupleobject.py @@ -227,6 +227,43 @@ else: module.set_after_use(s) - def test_torch(self): + def test_mp_length(self): + # issue 2968: creating a subclass of tuple in C led to recursion + # since the default tp_new needs to build a w_obj, but that needs + # to call space.len_w, which needs to call tp_new. module = self.import_module('THPSize') - module.Size() + module = self.import_extension('foo', [ + ("get_size", "METH_NOARGS", + """ + return (PyObject*)&THPSizeType; + """), + ], prologue=''' + #include "Python.h" + + struct THPSize { + PyTupleObject tuple; + } THPSize; + + static PyMappingMethods THPSize_as_mapping = { + 0, //PyTuple_Type.tp_as_mapping->mp_length, + 0, + 0 + }; + + PyTypeObject THPSizeType = { + PyVarObject_HEAD_INIT(0, 0) + "torch.Size", /* tp_name */ + sizeof(THPSize), /* tp_basicsize */ + }; + ''' , more_init = ''' + THPSize_as_mapping.mp_length = PyTuple_Type.tp_as_mapping->mp_length; + THPSizeType.tp_base = &PyTuple_Type; + THPSizeType.tp_flags = Py_TPFLAGS_DEFAULT; + THPSizeType.tp_as_mapping = &THPSize_as_mapping; + THPSizeType.tp_new = PyTuple_Type.tp_new; + if (PyType_Ready(&THPSizeType) < 0) INITERROR; + ''') + SZ = module.get_size() + s = SZ((1, 2, 3)) + assert len(s) == 3 + diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -687,6 +687,7 @@ else: update_all_slots_builtin(space, w_type, pto) + # XXX generlize this pattern for various slot functions implemented in C if space.is_w(w_type, space.w_tuple): pto.c_tp_new = state.C.tuple_new From pypy.commits at gmail.com Thu Mar 14 11:18:57 2019 From: pypy.commits at gmail.com (mattip) Date: Thu, 14 Mar 2019 08:18:57 -0700 (PDT) Subject: [pypy-commit] pypy default: move PyTuple_Type.tp_new to C Message-ID: <5c8a70e1.1c69fb81.46511.13db@mx.google.com> Author: Matti Picus Branch: Changeset: r96311:23b9c2f3f1d0 Date: 2019-03-10 21:54 +0200 http://bitbucket.org/pypy/pypy/changeset/23b9c2f3f1d0/ Log: move PyTuple_Type.tp_new to C diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -1188,7 +1188,9 @@ state.C.get_pyos_inputhook = rffi.llexternal( '_PyPy_get_PyOS_InputHook', [], FUNCPTR, compilation_info=eci, _nowrapper=True) - + state.C.tuple_new = rffi.llexternal( + 'tuple_new', [PyTypeObjectPtr, PyObject, PyObject], PyObject, + compilation_info=eci, _nowrapper=True) def init_function(func): INIT_FUNCTIONS.append(func) diff --git a/pypy/module/cpyext/include/tupleobject.h b/pypy/module/cpyext/include/tupleobject.h --- a/pypy/module/cpyext/include/tupleobject.h +++ b/pypy/module/cpyext/include/tupleobject.h @@ -18,6 +18,7 @@ PyAPI_FUNC(PyObject *) PyTuple_New(Py_ssize_t size); PyAPI_FUNC(void) _PyPy_tuple_dealloc(PyObject *); +PyAPI_FUNC(PyObject *) tuple_new(PyTypeObject *type, PyObject *args, PyObject *kwds); /* defined in varargswrapper.c */ PyAPI_FUNC(PyObject *) PyTuple_Pack(Py_ssize_t, ...); diff --git a/pypy/module/cpyext/src/tupleobject.c b/pypy/module/cpyext/src/tupleobject.c --- a/pypy/module/cpyext/src/tupleobject.c +++ b/pypy/module/cpyext/src/tupleobject.c @@ -89,3 +89,48 @@ done: Py_TRASHCAN_SAFE_END(op) } + +static PyObject * +tuple_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds); + +PyObject * +tuple_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyObject *arg = NULL; + static char *kwlist[] = {"sequence", 0}; + + if (type != &PyTuple_Type) + return tuple_subtype_new(type, args, kwds); + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O:tuple", kwlist, &arg)) + return NULL; + + if (arg == NULL) + return PyTuple_New(0); + else + return PySequence_Tuple(arg); +} + +static PyObject * +tuple_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyObject *tmp, *newobj, *item; + Py_ssize_t i, n; + + assert(PyType_IsSubtype(type, &PyTuple_Type)); + tmp = tuple_new(&PyTuple_Type, args, kwds); + if (tmp == NULL) + return NULL; + assert(PyTuple_Check(tmp)); + newobj = type->tp_alloc(type, n = PyTuple_GET_SIZE(tmp)); + if (newobj == NULL) + return NULL; + for (i = 0; i < n; i++) { + item = PyTuple_GET_ITEM(tmp, i); + Py_INCREF(item); + PyTuple_SET_ITEM(newobj, i, item); + } + Py_DECREF(tmp); + return newobj; +} + + diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -796,6 +796,10 @@ update_all_slots(space, w_type, pto) else: update_all_slots_builtin(space, w_type, pto) + + if space.is_w(w_type, space.w_tuple): + pto.c_tp_new = state.C.tuple_new + if not pto.c_tp_new: base_object_pyo = make_ref(space, space.w_object) base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo) From pypy.commits at gmail.com Thu Mar 14 11:18:59 2019 From: pypy.commits at gmail.com (mattip) Date: Thu, 14 Mar 2019 08:18:59 -0700 (PDT) Subject: [pypy-commit] pypy default: rework test for python2, python3 Message-ID: <5c8a70e3.1c69fb81.4f9bb.0fbc@mx.google.com> Author: Matti Picus Branch: Changeset: r96312:da49044a5452 Date: 2019-03-14 17:03 +0200 http://bitbucket.org/pypy/pypy/changeset/da49044a5452/ Log: rework test for python2, python3 diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py --- a/pypy/module/cpyext/test/test_tupleobject.py +++ b/pypy/module/cpyext/test/test_tupleobject.py @@ -226,3 +226,43 @@ raises(SystemError, module.set_after_use, s) else: module.set_after_use(s) + + def test_mp_length(self): + # issue 2968: creating a subclass of tuple in C led to recursion + # since the default tp_new needs to build a w_obj, but that needs + # to call space.len_w, which needs to call tp_new. + module = self.import_module('THPSize') + module = self.import_extension('foo', [ + ("get_size", "METH_NOARGS", + """ + return (PyObject*)&THPSizeType; + """), + ], prologue=''' + #include "Python.h" + + struct THPSize { + PyTupleObject tuple; + } THPSize; + + static PyMappingMethods THPSize_as_mapping = { + 0, //PyTuple_Type.tp_as_mapping->mp_length, + 0, + 0 + }; + + PyTypeObject THPSizeType = { + PyVarObject_HEAD_INIT(0, 0) + "torch.Size", /* tp_name */ + sizeof(THPSize), /* tp_basicsize */ + }; + ''' , more_init = ''' + THPSize_as_mapping.mp_length = PyTuple_Type.tp_as_mapping->mp_length; + THPSizeType.tp_base = &PyTuple_Type; + THPSizeType.tp_flags = Py_TPFLAGS_DEFAULT; + THPSizeType.tp_as_mapping = &THPSize_as_mapping; + THPSizeType.tp_new = PyTuple_Type.tp_new; + if (PyType_Ready(&THPSizeType) < 0) INITERROR; + ''') + SZ = module.get_size() + s = SZ((1, 2, 3)) + assert len(s) == 3 diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -797,6 +797,7 @@ else: update_all_slots_builtin(space, w_type, pto) + # XXX generlize this pattern for various slot functions implemented in C if space.is_w(w_type, space.w_tuple): pto.c_tp_new = state.C.tuple_new From pypy.commits at gmail.com Thu Mar 14 11:19:01 2019 From: pypy.commits at gmail.com (mattip) Date: Thu, 14 Mar 2019 08:19:01 -0700 (PDT) Subject: [pypy-commit] pypy default: fix merge from grafting issue2968 Message-ID: <5c8a70e5.1c69fb81.6a719.c9e7@mx.google.com> Author: Matti Picus Branch: Changeset: r96313:d3aefbf6dae7 Date: 2019-03-14 17:15 +0200 http://bitbucket.org/pypy/pypy/changeset/d3aefbf6dae7/ Log: fix merge from grafting issue2968 diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py --- a/pypy/module/cpyext/test/test_tupleobject.py +++ b/pypy/module/cpyext/test/test_tupleobject.py @@ -231,7 +231,6 @@ # issue 2968: creating a subclass of tuple in C led to recursion # since the default tp_new needs to build a w_obj, but that needs # to call space.len_w, which needs to call tp_new. - module = self.import_module('THPSize') module = self.import_extension('foo', [ ("get_size", "METH_NOARGS", """ From pypy.commits at gmail.com Thu Mar 14 11:19:02 2019 From: pypy.commits at gmail.com (mattip) Date: Thu, 14 Mar 2019 08:19:02 -0700 (PDT) Subject: [pypy-commit] pypy issue2968: close branch to be merged Message-ID: <5c8a70e6.1c69fb81.8dafc.533d@mx.google.com> Author: Matti Picus Branch: issue2968 Changeset: r96314:074e102b8636 Date: 2019-03-14 17:16 +0200 http://bitbucket.org/pypy/pypy/changeset/074e102b8636/ Log: close branch to be merged From pypy.commits at gmail.com Thu Mar 14 11:19:04 2019 From: pypy.commits at gmail.com (mattip) Date: Thu, 14 Mar 2019 08:19:04 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: merge issue 2968 into py3.6 Message-ID: <5c8a70e8.1c69fb81.e643c.5e81@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r96315:12d0c4f992fd Date: 2019-03-14 17:17 +0200 http://bitbucket.org/pypy/pypy/changeset/12d0c4f992fd/ Log: merge issue 2968 into py3.6 diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -1191,7 +1191,9 @@ state.C.get_pyos_inputhook = rffi.llexternal( '_PyPy_get_PyOS_InputHook', [], FUNCPTR, compilation_info=eci, _nowrapper=True) - + state.C.tuple_new = rffi.llexternal( + 'tuple_new', [PyTypeObjectPtr, PyObject, PyObject], PyObject, + compilation_info=eci, _nowrapper=True) def init_function(func): INIT_FUNCTIONS.append(func) diff --git a/pypy/module/cpyext/include/tupleobject.h b/pypy/module/cpyext/include/tupleobject.h --- a/pypy/module/cpyext/include/tupleobject.h +++ b/pypy/module/cpyext/include/tupleobject.h @@ -18,6 +18,7 @@ PyAPI_FUNC(PyObject *) PyTuple_New(Py_ssize_t size); PyAPI_FUNC(void) _PyPy_tuple_dealloc(PyObject *); +PyAPI_FUNC(PyObject *) tuple_new(PyTypeObject *type, PyObject *args, PyObject *kwds); /* defined in varargswrapper.c */ PyAPI_FUNC(PyObject *) PyTuple_Pack(Py_ssize_t, ...); diff --git a/pypy/module/cpyext/src/tupleobject.c b/pypy/module/cpyext/src/tupleobject.c --- a/pypy/module/cpyext/src/tupleobject.c +++ b/pypy/module/cpyext/src/tupleobject.c @@ -89,3 +89,48 @@ done: Py_TRASHCAN_SAFE_END(op) } + +static PyObject * +tuple_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds); + +PyObject * +tuple_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyObject *arg = NULL; + static char *kwlist[] = {"sequence", 0}; + + if (type != &PyTuple_Type) + return tuple_subtype_new(type, args, kwds); + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O:tuple", kwlist, &arg)) + return NULL; + + if (arg == NULL) + return PyTuple_New(0); + else + return PySequence_Tuple(arg); +} + +static PyObject * +tuple_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyObject *tmp, *newobj, *item; + Py_ssize_t i, n; + + assert(PyType_IsSubtype(type, &PyTuple_Type)); + tmp = tuple_new(&PyTuple_Type, args, kwds); + if (tmp == NULL) + return NULL; + assert(PyTuple_Check(tmp)); + newobj = type->tp_alloc(type, n = PyTuple_GET_SIZE(tmp)); + if (newobj == NULL) + return NULL; + for (i = 0; i < n; i++) { + item = PyTuple_GET_ITEM(tmp, i); + Py_INCREF(item); + PyTuple_SET_ITEM(newobj, i, item); + } + Py_DECREF(tmp); + return newobj; +} + + diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py --- a/pypy/module/cpyext/test/test_tupleobject.py +++ b/pypy/module/cpyext/test/test_tupleobject.py @@ -226,3 +226,44 @@ raises(SystemError, module.set_after_use, s) else: module.set_after_use(s) + + def test_mp_length(self): + # issue 2968: creating a subclass of tuple in C led to recursion + # since the default tp_new needs to build a w_obj, but that needs + # to call space.len_w, which needs to call tp_new. + module = self.import_module('THPSize') + module = self.import_extension('foo', [ + ("get_size", "METH_NOARGS", + """ + return (PyObject*)&THPSizeType; + """), + ], prologue=''' + #include "Python.h" + + struct THPSize { + PyTupleObject tuple; + } THPSize; + + static PyMappingMethods THPSize_as_mapping = { + 0, //PyTuple_Type.tp_as_mapping->mp_length, + 0, + 0 + }; + + PyTypeObject THPSizeType = { + PyVarObject_HEAD_INIT(0, 0) + "torch.Size", /* tp_name */ + sizeof(THPSize), /* tp_basicsize */ + }; + ''' , more_init = ''' + THPSize_as_mapping.mp_length = PyTuple_Type.tp_as_mapping->mp_length; + THPSizeType.tp_base = &PyTuple_Type; + THPSizeType.tp_flags = Py_TPFLAGS_DEFAULT; + THPSizeType.tp_as_mapping = &THPSize_as_mapping; + THPSizeType.tp_new = PyTuple_Type.tp_new; + if (PyType_Ready(&THPSizeType) < 0) INITERROR; + ''') + SZ = module.get_size() + s = SZ((1, 2, 3)) + assert len(s) == 3 + diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -686,6 +686,11 @@ update_all_slots(space, w_type, pto) else: update_all_slots_builtin(space, w_type, pto) + + # XXX generlize this pattern for various slot functions implemented in C + if space.is_w(w_type, space.w_tuple): + pto.c_tp_new = state.C.tuple_new + if not pto.c_tp_new: base_object_pyo = make_ref(space, space.w_object) base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo) From pypy.commits at gmail.com Thu Mar 14 11:19:06 2019 From: pypy.commits at gmail.com (mattip) Date: Thu, 14 Mar 2019 08:19:06 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: merge default into branch Message-ID: <5c8a70ea.1c69fb81.818fc.65f1@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r96316:605f74b0221c Date: 2019-03-14 17:18 +0200 http://bitbucket.org/pypy/pypy/changeset/605f74b0221c/ Log: merge default into branch diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py --- a/pypy/module/cpyext/test/test_tupleobject.py +++ b/pypy/module/cpyext/test/test_tupleobject.py @@ -231,7 +231,6 @@ # issue 2968: creating a subclass of tuple in C led to recursion # since the default tp_new needs to build a w_obj, but that needs # to call space.len_w, which needs to call tp_new. - module = self.import_module('THPSize') module = self.import_extension('foo', [ ("get_size", "METH_NOARGS", """ @@ -266,4 +265,5 @@ SZ = module.get_size() s = SZ((1, 2, 3)) assert len(s) == 3 + assert len(s) == 3 diff --git a/rpython/rlib/test/test_rawrefcount_boehm.py b/rpython/rlib/test/test_rawrefcount_boehm.py --- a/rpython/rlib/test/test_rawrefcount_boehm.py +++ b/rpython/rlib/test/test_rawrefcount_boehm.py @@ -111,7 +111,7 @@ pyobjs.append(varname) return varname - for op in draw(strategies.lists(operations, average_size=250)): + for op in draw(strategies.lists(operations)): if op == 'new_gcobj': new_gcobj() elif op == 'new_pyobj': From pypy.commits at gmail.com Thu Mar 14 11:20:25 2019 From: pypy.commits at gmail.com (mattip) Date: Thu, 14 Mar 2019 08:20:25 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy2.7-v7.x: merge default into release Message-ID: <5c8a7139.1c69fb81.988a2.9c27@mx.google.com> Author: Matti Picus Branch: release-pypy2.7-v7.x Changeset: r96317:990cef41fe11 Date: 2019-03-14 17:19 +0200 http://bitbucket.org/pypy/pypy/changeset/990cef41fe11/ Log: merge default into release diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -306,8 +306,8 @@ return w_err_handler - at unwrap_spec(errors='text') -def encode(space, w_obj, w_encoding=None, errors='strict'): + at unwrap_spec(encoding='text_or_none', errors='text_or_none') +def encode(space, w_obj, encoding=None, errors=None): """encode(obj, [encoding[,errors]]) -> object Encodes obj using the codec registered for encoding. encoding defaults @@ -317,13 +317,19 @@ 'xmlcharrefreplace' as well as any other name registered with codecs.register_error that can handle ValueErrors. """ - if w_encoding is None: + if encoding is None: encoding = space.sys.defaultencoding + w_encoder = space.getitem(lookup_codec(space, encoding), space.newint(0)) + if errors: + w_res = space.call_function(w_encoder, w_obj, space.newtext(errors)) else: - encoding = space.text_w(w_encoding) - w_encoder = space.getitem(lookup_codec(space, encoding), space.newint(0)) - w_res = space.call_function(w_encoder, w_obj, space.newtext(errors)) - return space.getitem(w_res, space.newint(0)) + w_res = space.call_function(w_encoder, w_obj) + w_retval = space.getitem(w_res, space.newint(0)) + if not space.isinstance_w(w_retval, space.w_bytes): + raise oefmt(space.w_TypeError, + "encoder did not return an string object (type '%T')", + w_retval) + return w_retval @unwrap_spec(errors='text_or_none') def readbuffer_encode(space, w_data, errors='strict'): @@ -335,8 +341,8 @@ s = space.getarg_w('t#', w_data) return space.newtuple([space.newbytes(s), space.newint(len(s))]) - at unwrap_spec(errors='text') -def decode(space, w_obj, w_encoding=None, errors='strict'): + at unwrap_spec(encoding='text_or_none', errors='text_or_none') +def decode(space, w_obj, encoding=None, errors=None): """decode(obj, [encoding[,errors]]) -> object Decodes obj using the codec registered for encoding. encoding defaults @@ -346,19 +352,17 @@ as well as any other name registered with codecs.register_error that is able to handle ValueErrors. """ - if w_encoding is None: + if encoding is None: encoding = space.sys.defaultencoding + w_decoder = space.getitem(lookup_codec(space, encoding), space.newint(1)) + if errors: + w_res = space.call_function(w_decoder, w_obj, space.newtext(errors)) else: - encoding = space.text_w(w_encoding) - w_decoder = space.getitem(lookup_codec(space, encoding), space.newint(1)) - if space.is_true(w_decoder): - w_res = space.call_function(w_decoder, w_obj, space.newtext(errors)) - if (not space.isinstance_w(w_res, space.w_tuple) or space.len_w(w_res) != 2): - raise oefmt(space.w_TypeError, - "encoder must return a tuple (object, integer)") - return space.getitem(w_res, space.newint(0)) - else: - assert 0, "XXX, what to do here?" + w_res = space.call_function(w_decoder, w_obj) + if (not space.isinstance_w(w_res, space.w_tuple) or space.len_w(w_res) != 2): + raise oefmt(space.w_TypeError, + "encoder must return a tuple (object, integer)") + return space.getitem(w_res, space.newint(0)) @unwrap_spec(errors='text') def register_error(space, errors, w_handler): diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -468,8 +468,10 @@ return (encode_one, decode_one, None, None) return None _codecs.register(search_function) - assert u"hello".encode("onearg") == 'foo' - assert b"hello".decode("onearg") == 'foo' + assert u"hello".encode("onearg") == b'foo' + assert b"hello".decode("onearg") == u'foo' + assert _codecs.encode(u"hello", "onearg") == b'foo' + assert _codecs.decode(b"hello", "onearg") == u'foo' def test_cpytest_decode(self): import codecs diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -1188,7 +1188,9 @@ state.C.get_pyos_inputhook = rffi.llexternal( '_PyPy_get_PyOS_InputHook', [], FUNCPTR, compilation_info=eci, _nowrapper=True) - + state.C.tuple_new = rffi.llexternal( + 'tuple_new', [PyTypeObjectPtr, PyObject, PyObject], PyObject, + compilation_info=eci, _nowrapper=True) def init_function(func): INIT_FUNCTIONS.append(func) diff --git a/pypy/module/cpyext/include/tupleobject.h b/pypy/module/cpyext/include/tupleobject.h --- a/pypy/module/cpyext/include/tupleobject.h +++ b/pypy/module/cpyext/include/tupleobject.h @@ -18,6 +18,7 @@ PyAPI_FUNC(PyObject *) PyTuple_New(Py_ssize_t size); PyAPI_FUNC(void) _PyPy_tuple_dealloc(PyObject *); +PyAPI_FUNC(PyObject *) tuple_new(PyTypeObject *type, PyObject *args, PyObject *kwds); /* defined in varargswrapper.c */ PyAPI_FUNC(PyObject *) PyTuple_Pack(Py_ssize_t, ...); diff --git a/pypy/module/cpyext/src/tupleobject.c b/pypy/module/cpyext/src/tupleobject.c --- a/pypy/module/cpyext/src/tupleobject.c +++ b/pypy/module/cpyext/src/tupleobject.c @@ -89,3 +89,48 @@ done: Py_TRASHCAN_SAFE_END(op) } + +static PyObject * +tuple_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds); + +PyObject * +tuple_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyObject *arg = NULL; + static char *kwlist[] = {"sequence", 0}; + + if (type != &PyTuple_Type) + return tuple_subtype_new(type, args, kwds); + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O:tuple", kwlist, &arg)) + return NULL; + + if (arg == NULL) + return PyTuple_New(0); + else + return PySequence_Tuple(arg); +} + +static PyObject * +tuple_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyObject *tmp, *newobj, *item; + Py_ssize_t i, n; + + assert(PyType_IsSubtype(type, &PyTuple_Type)); + tmp = tuple_new(&PyTuple_Type, args, kwds); + if (tmp == NULL) + return NULL; + assert(PyTuple_Check(tmp)); + newobj = type->tp_alloc(type, n = PyTuple_GET_SIZE(tmp)); + if (newobj == NULL) + return NULL; + for (i = 0; i < n; i++) { + item = PyTuple_GET_ITEM(tmp, i); + Py_INCREF(item); + PyTuple_SET_ITEM(newobj, i, item); + } + Py_DECREF(tmp); + return newobj; +} + + diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py --- a/pypy/module/cpyext/test/test_tupleobject.py +++ b/pypy/module/cpyext/test/test_tupleobject.py @@ -226,3 +226,42 @@ raises(SystemError, module.set_after_use, s) else: module.set_after_use(s) + + def test_mp_length(self): + # issue 2968: creating a subclass of tuple in C led to recursion + # since the default tp_new needs to build a w_obj, but that needs + # to call space.len_w, which needs to call tp_new. + module = self.import_extension('foo', [ + ("get_size", "METH_NOARGS", + """ + return (PyObject*)&THPSizeType; + """), + ], prologue=''' + #include "Python.h" + + struct THPSize { + PyTupleObject tuple; + } THPSize; + + static PyMappingMethods THPSize_as_mapping = { + 0, //PyTuple_Type.tp_as_mapping->mp_length, + 0, + 0 + }; + + PyTypeObject THPSizeType = { + PyVarObject_HEAD_INIT(0, 0) + "torch.Size", /* tp_name */ + sizeof(THPSize), /* tp_basicsize */ + }; + ''' , more_init = ''' + THPSize_as_mapping.mp_length = PyTuple_Type.tp_as_mapping->mp_length; + THPSizeType.tp_base = &PyTuple_Type; + THPSizeType.tp_flags = Py_TPFLAGS_DEFAULT; + THPSizeType.tp_as_mapping = &THPSize_as_mapping; + THPSizeType.tp_new = PyTuple_Type.tp_new; + if (PyType_Ready(&THPSizeType) < 0) INITERROR; + ''') + SZ = module.get_size() + s = SZ((1, 2, 3)) + assert len(s) == 3 diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -796,6 +796,11 @@ update_all_slots(space, w_type, pto) else: update_all_slots_builtin(space, w_type, pto) + + # XXX generlize this pattern for various slot functions implemented in C + if space.is_w(w_type, space.w_tuple): + pto.c_tp_new = state.C.tuple_new + if not pto.c_tp_new: base_object_pyo = make_ref(space, space.w_object) base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo) diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -1071,16 +1071,17 @@ return encoding, errors -def encode_object(space, w_object, encoding, errors): - w_encoder = None +def encode_object(space, w_obj, encoding, errors): + from pypy.module._codecs.interp_codecs import encode if errors is None or errors == 'strict': + # fast path if ((encoding is None and space.sys.defaultencoding == 'ascii') or encoding == 'ascii'): - s = space.utf8_w(w_object) + s = space.utf8_w(w_obj) try: rutf8.check_ascii(s) except rutf8.CheckError as a: - if space.isinstance_w(w_object, space.w_unicode): + if space.isinstance_w(w_obj, space.w_unicode): eh = unicodehelper.encode_error_handler(space) else: # must be a bytes-like object. In order to encode it, @@ -1093,32 +1094,17 @@ return space.newbytes(s) if ((encoding is None and space.sys.defaultencoding == 'utf8') or encoding == 'utf-8' or encoding == 'utf8' or encoding == 'UTF-8'): - utf8 = space.utf8_w(w_object) + utf8 = space.utf8_w(w_obj) if rutf8.has_surrogates(utf8): utf8 = rutf8.reencode_utf8_with_surrogates(utf8) return space.newbytes(utf8) - if encoding is None: - # Get the encoder functions as a wrapped object. - # This lookup is cached. - w_encoder = space.sys.get_w_default_encoder() - if w_encoder is None: - from pypy.module._codecs.interp_codecs import lookup_codec - w_encoder = space.getitem(lookup_codec(space, encoding), space.newint(0)) - if errors is None: - w_restuple = space.call_function(w_encoder, w_object) - else: - w_errors = space.newtext(errors) - w_restuple = space.call_function(w_encoder, w_object, w_errors) - w_retval = space.getitem(w_restuple, space.newint(0)) - if not space.isinstance_w(w_retval, space.w_bytes): - raise oefmt(space.w_TypeError, - "encoder did not return an string object (type '%T')", - w_retval) - return w_retval + return encode(space, w_obj, encoding, errors) def decode_object(space, w_obj, encoding, errors): + from pypy.module._codecs.interp_codecs import lookup_codec, decode if errors is None or errors == 'strict': + # fast paths if encoding is None: encoding = getdefaultencoding(space) if encoding == 'ascii': @@ -1133,20 +1119,9 @@ s = space.charbuf_w(w_obj) lgt = unicodehelper.check_utf8_or_raise(space, s) return space.newutf8(s, lgt) - w_decoder = None if encoding is None: - # Get the decoder functions as a wrapped object. - # This lookup is cached. - w_decoder = space.sys.get_w_default_decoder() - if w_decoder is None: - from pypy.module._codecs.interp_codecs import lookup_codec - w_decoder = space.getitem(lookup_codec(space, encoding), space.newint(1)) - if errors is None: - w_retval = space.call_function(w_decoder, w_obj) - else: - w_retval = space.call_function(w_decoder, w_obj, space.newtext(errors)) - return space.getitem(w_retval, space.newint(0)) - + encoding = space.sys.defaultencoding + return decode(space, w_obj, encoding, errors) def unicode_from_encoded_object(space, w_obj, encoding, errors): # explicitly block bytearray on 2.7 diff --git a/rpython/rlib/rsre/rsre_utf8.py b/rpython/rlib/rsre/rsre_utf8.py --- a/rpython/rlib/rsre/rsre_utf8.py +++ b/rpython/rlib/rsre/rsre_utf8.py @@ -40,17 +40,23 @@ prev_indirect = prev def next_n(self, position, n, end_position): - for i in range(n): + i = 0 + # avoid range(n) since n can be quite large + while i < n: if position >= end_position: raise EndOfString position = rutf8.next_codepoint_pos(self._utf8, position) + i += 1 return position def prev_n(self, position, n, start_position): - for i in range(n): + i = 0 + # avoid range(n) since n can be quite large + while i < n: if position <= start_position: raise EndOfString position = rutf8.prev_codepoint_pos(self._utf8, position) + i += 1 assert position >= 0 return position diff --git a/rpython/rlib/test/test_rawrefcount_boehm.py b/rpython/rlib/test/test_rawrefcount_boehm.py --- a/rpython/rlib/test/test_rawrefcount_boehm.py +++ b/rpython/rlib/test/test_rawrefcount_boehm.py @@ -111,7 +111,7 @@ pyobjs.append(varname) return varname - for op in draw(strategies.lists(operations, average_size=250)): + for op in draw(strategies.lists(operations)): if op == 'new_gcobj': new_gcobj() elif op == 'new_pyobj': From pypy.commits at gmail.com Thu Mar 14 11:20:28 2019 From: pypy.commits at gmail.com (mattip) Date: Thu, 14 Mar 2019 08:20:28 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy3.6-v7.x: merge py3.6 into release Message-ID: <5c8a713c.1c69fb81.dc544.6eeb@mx.google.com> Author: Matti Picus Branch: release-pypy3.6-v7.x Changeset: r96318:bb0d05b190b9 Date: 2019-03-14 17:19 +0200 http://bitbucket.org/pypy/pypy/changeset/bb0d05b190b9/ Log: merge py3.6 into release diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst --- a/pypy/doc/whatsnew-pypy3-head.rst +++ b/pypy/doc/whatsnew-pypy3-head.rst @@ -4,3 +4,7 @@ .. this is the revision after release-pypy3.6-v7.1 .. startrev: d642a3c217cb + +.. branch: zlib-make-py3-go-boom + +Complain if you try to copy a flushed zlib decompress on py3 diff --git a/pypy/module/__pypy__/test/test_newmemoryview.py b/pypy/module/__pypy__/test/test_newmemoryview.py --- a/pypy/module/__pypy__/test/test_newmemoryview.py +++ b/pypy/module/__pypy__/test/test_newmemoryview.py @@ -26,7 +26,7 @@ obj = B() - buf = buffer(obj) + buf = memoryview(obj) v = obj.data[2] - assert ord(buf[2]) == v + assert buf[2] == v diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -304,7 +304,7 @@ while pos < end: oc = ord(obj[pos]) raw_unicode_escape_helper(builder, oc) - pos += 1 + pos += 1 return space.newtuple([space.newtext(builder.build()), w_end]) else: raise oefmt(space.w_TypeError, @@ -561,8 +561,8 @@ return w_err_handler - at unwrap_spec(errors='text') -def encode(space, w_obj, w_encoding=None, errors='strict'): + at unwrap_spec(encoding='text_or_none', errors='text_or_none') +def encode(space, w_obj, encoding=None, errors=None): """encode(obj, [encoding[,errors]]) -> object Encodes obj using the codec registered for encoding. encoding defaults @@ -572,20 +572,26 @@ 'xmlcharrefreplace' as well as any other name registered with codecs.register_error that can handle ValueErrors. """ - if w_encoding is None: + if encoding is None: encoding = space.sys.defaultencoding - else: - encoding = space.text_w(w_encoding) w_encoder = space.getitem(lookup_codec(space, encoding), space.newint(0)) - return _call_codec(space, w_encoder, w_obj, "encoding", encoding, errors) + w_retval = _call_codec(space, w_encoder, w_obj, "encoding", encoding, errors) + if not space.isinstance_w(w_retval, space.w_bytes): + raise oefmt(space.w_TypeError, + "'%s' encoder returned '%T' instead of 'bytes'; " + "use codecs.encode() to encode to arbitrary types", + encoding, + w_retval) + return w_retval @unwrap_spec(errors='text_or_none') def readbuffer_encode(space, w_data, errors='strict'): s = space.getarg_w('s#', w_data) return space.newtuple([space.newbytes(s), space.newint(len(s))]) - at unwrap_spec(errors='text') -def decode(space, w_obj, w_encoding=None, errors='strict'): + at unwrap_spec(encoding='text_or_none', errors='text_or_none') +def decode(space, w_obj, encoding=None, errors=None): + from pypy.objspace.std.unicodeobject import W_UnicodeObject """decode(obj, [encoding[,errors]]) -> object Decodes obj using the codec registered for encoding. encoding defaults @@ -595,12 +601,17 @@ as well as any other name registered with codecs.register_error that is able to handle ValueErrors. """ - if w_encoding is None: + if encoding is None: encoding = space.sys.defaultencoding - else: - encoding = space.text_w(w_encoding) w_decoder = space.getitem(lookup_codec(space, encoding), space.newint(1)) - return _call_codec(space, w_decoder, w_obj, "decoding", encoding, errors) + w_retval = _call_codec(space, w_decoder, w_obj, "decoding", encoding, errors) + if not isinstance(w_retval, W_UnicodeObject): + raise oefmt(space.w_TypeError, + "'%s' decoder returned '%T' instead of 'str'; " + "use codecs.decode() to decode to arbitrary types", + encoding, + w_retval) + return w_retval @unwrap_spec(errors='text') def register_error(space, errors, w_handler): @@ -636,16 +647,6 @@ "use %s to handle arbitrary codecs", encoding, action) return codec_info -def encode_text(space, w_obj, encoding, errors): - w_encoder = space.getitem( - lookup_text_codec(space, "codecs.encode()", encoding), space.newint(0)) - return _call_codec(space, w_encoder, w_obj, "encoding", encoding, errors) - -def decode_text(space, w_obj, encoding, errors): - w_decoder = space.getitem( - lookup_text_codec(space, "codecs.decode()", encoding), space.newint(1)) - return _call_codec(space, w_decoder, w_obj, "decoding", encoding, errors) - # ____________________________________________________________ def _find_implementation(impl_name): @@ -735,7 +736,7 @@ result = unicodehelper.utf8_encode_utf_8(utf8, errors, state.encode_error_handler, allow_surrogates=False) except unicodehelper.ErrorHandlerError as e: - raise oefmt(space.w_IndexError, + raise oefmt(space.w_IndexError, "position %d from error handler invalid, already encoded %d", e.new,e.old) diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -708,7 +708,9 @@ return None _codecs.register(search_function) assert u"hello".encode("onearg") == b'foo' - assert b"hello".decode("onearg") == 'foo' + assert b"hello".decode("onearg") == u'foo' + assert _codecs.encode(u"hello", "onearg") == b'foo' + assert _codecs.decode(b"hello", "onearg") == u'foo' def test_cpytest_decode(self): import codecs diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -1191,7 +1191,9 @@ state.C.get_pyos_inputhook = rffi.llexternal( '_PyPy_get_PyOS_InputHook', [], FUNCPTR, compilation_info=eci, _nowrapper=True) - + state.C.tuple_new = rffi.llexternal( + 'tuple_new', [PyTypeObjectPtr, PyObject, PyObject], PyObject, + compilation_info=eci, _nowrapper=True) def init_function(func): INIT_FUNCTIONS.append(func) diff --git a/pypy/module/cpyext/include/tupleobject.h b/pypy/module/cpyext/include/tupleobject.h --- a/pypy/module/cpyext/include/tupleobject.h +++ b/pypy/module/cpyext/include/tupleobject.h @@ -18,6 +18,7 @@ PyAPI_FUNC(PyObject *) PyTuple_New(Py_ssize_t size); PyAPI_FUNC(void) _PyPy_tuple_dealloc(PyObject *); +PyAPI_FUNC(PyObject *) tuple_new(PyTypeObject *type, PyObject *args, PyObject *kwds); /* defined in varargswrapper.c */ PyAPI_FUNC(PyObject *) PyTuple_Pack(Py_ssize_t, ...); diff --git a/pypy/module/cpyext/src/tupleobject.c b/pypy/module/cpyext/src/tupleobject.c --- a/pypy/module/cpyext/src/tupleobject.c +++ b/pypy/module/cpyext/src/tupleobject.c @@ -89,3 +89,48 @@ done: Py_TRASHCAN_SAFE_END(op) } + +static PyObject * +tuple_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds); + +PyObject * +tuple_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyObject *arg = NULL; + static char *kwlist[] = {"sequence", 0}; + + if (type != &PyTuple_Type) + return tuple_subtype_new(type, args, kwds); + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O:tuple", kwlist, &arg)) + return NULL; + + if (arg == NULL) + return PyTuple_New(0); + else + return PySequence_Tuple(arg); +} + +static PyObject * +tuple_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyObject *tmp, *newobj, *item; + Py_ssize_t i, n; + + assert(PyType_IsSubtype(type, &PyTuple_Type)); + tmp = tuple_new(&PyTuple_Type, args, kwds); + if (tmp == NULL) + return NULL; + assert(PyTuple_Check(tmp)); + newobj = type->tp_alloc(type, n = PyTuple_GET_SIZE(tmp)); + if (newobj == NULL) + return NULL; + for (i = 0; i < n; i++) { + item = PyTuple_GET_ITEM(tmp, i); + Py_INCREF(item); + PyTuple_SET_ITEM(newobj, i, item); + } + Py_DECREF(tmp); + return newobj; +} + + diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py --- a/pypy/module/cpyext/test/test_tupleobject.py +++ b/pypy/module/cpyext/test/test_tupleobject.py @@ -226,3 +226,44 @@ raises(SystemError, module.set_after_use, s) else: module.set_after_use(s) + + def test_mp_length(self): + # issue 2968: creating a subclass of tuple in C led to recursion + # since the default tp_new needs to build a w_obj, but that needs + # to call space.len_w, which needs to call tp_new. + module = self.import_extension('foo', [ + ("get_size", "METH_NOARGS", + """ + return (PyObject*)&THPSizeType; + """), + ], prologue=''' + #include "Python.h" + + struct THPSize { + PyTupleObject tuple; + } THPSize; + + static PyMappingMethods THPSize_as_mapping = { + 0, //PyTuple_Type.tp_as_mapping->mp_length, + 0, + 0 + }; + + PyTypeObject THPSizeType = { + PyVarObject_HEAD_INIT(0, 0) + "torch.Size", /* tp_name */ + sizeof(THPSize), /* tp_basicsize */ + }; + ''' , more_init = ''' + THPSize_as_mapping.mp_length = PyTuple_Type.tp_as_mapping->mp_length; + THPSizeType.tp_base = &PyTuple_Type; + THPSizeType.tp_flags = Py_TPFLAGS_DEFAULT; + THPSizeType.tp_as_mapping = &THPSize_as_mapping; + THPSizeType.tp_new = PyTuple_Type.tp_new; + if (PyType_Ready(&THPSizeType) < 0) INITERROR; + ''') + SZ = module.get_size() + s = SZ((1, 2, 3)) + assert len(s) == 3 + assert len(s) == 3 + diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -686,6 +686,11 @@ update_all_slots(space, w_type, pto) else: update_all_slots_builtin(space, w_type, pto) + + # XXX generlize this pattern for various slot functions implemented in C + if space.is_w(w_type, space.w_tuple): + pto.c_tp_new = state.C.tuple_new + if not pto.c_tp_new: base_object_pyo = make_ref(space, space.w_object) base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo) diff --git a/pypy/module/zlib/interp_zlib.py b/pypy/module/zlib/interp_zlib.py --- a/pypy/module/zlib/interp_zlib.py +++ b/pypy/module/zlib/interp_zlib.py @@ -313,6 +313,11 @@ try: self.lock() try: + if not self.stream: + raise oefmt( + space.w_ValueError, + "Decompressor was already flushed", + ) copied = rzlib.inflateCopy(self.stream) finally: self.unlock() diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -43,7 +43,7 @@ # best effort, too expensive to handle surrogates ulength = rutf8.codepoints_in_utf(utf8str) except: - ulength = length + ulength = length assert ulength == length @@ -135,7 +135,7 @@ if strict: raise oefmt(space.w_TypeError, "%s arg must be None, unicode or str", strict) - return unicode_from_encoded_object(space, w_other, 'utf8', "strict") + return decode_object(space, w_other, 'utf8', "strict") def convert_to_w_unicode(self, space): return self @@ -203,8 +203,7 @@ if space.isinstance_w(w_object, space.w_unicode): raise oefmt(space.w_TypeError, "decoding str is not supported") - w_value = unicode_from_encoded_object(space, w_object, - encoding, errors) + w_value = decode_object(space, w_object, encoding, errors) if space.is_w(w_unicodetype, space.w_unicode): return w_value @@ -649,7 +648,7 @@ def descr_startswith(self, space, w_prefix, w_start=None, w_end=None): start, end = self._unwrap_and_compute_idx_params(space, w_start, w_end) value = self._utf8 - if (start > 0 and not space.is_none(w_end) and + if (start > 0 and not space.is_none(w_end) and space.getindex_w(w_end, None) == 0): return space.w_False if space.isinstance_w(w_prefix, space.w_tuple): @@ -674,7 +673,7 @@ start, end = self._unwrap_and_compute_idx_params(space, w_start, w_end) value = self._utf8 # match cpython behaviour - if (start > 0 and not space.is_none(w_end) and + if (start > 0 and not space.is_none(w_end) and space.getindex_w(w_end, None) == 0): return space.w_False if space.isinstance_w(w_suffix, space.w_tuple): @@ -1207,11 +1206,11 @@ errors = None if w_errors is None else space.text_w(w_errors) return encoding, errors - -def encode_object(space, w_object, encoding, errors): - from pypy.module._codecs.interp_codecs import encode_text +def encode_object(space, w_obj, encoding, errors): + from pypy.module._codecs.interp_codecs import encode if errors is None or errors == 'strict': - utf8 = space.utf8_w(w_object) + # fast paths + utf8 = space.utf8_w(w_obj) if encoding is None or encoding == 'utf-8': try: rutf8.check_utf8(utf8, False) @@ -1230,21 +1229,12 @@ a.pos, a.pos + 1) assert False, "always raises" return space.newbytes(utf8) - if encoding is None: - encoding = space.sys.defaultencoding - w_retval = encode_text(space, w_object, encoding, errors) - if not space.isinstance_w(w_retval, space.w_bytes): - raise oefmt(space.w_TypeError, - "'%s' encoder returned '%T' instead of 'bytes'; " - "use codecs.encode() to encode to arbitrary types", - encoding, - w_retval) - return w_retval + return encode(space, w_obj, encoding, errors) def decode_object(space, w_obj, encoding, errors=None): - assert encoding is not None if errors == 'strict' or errors is None: + # fast paths if encoding == 'ascii': s = space.charbuf_w(w_obj) unicodehelper.check_ascii_or_raise(space, s) @@ -1253,27 +1243,8 @@ s = space.charbuf_w(w_obj) lgt = unicodehelper.check_utf8_or_raise(space, s) return space.newutf8(s, lgt) - from pypy.module._codecs.interp_codecs import decode_text - w_retval = decode_text(space, w_obj, encoding, errors) - if not isinstance(w_retval, W_UnicodeObject): - raise oefmt(space.w_TypeError, - "'%s' decoder returned '%T' instead of 'str'; " - "use codecs.decode() to decode to arbitrary types", - encoding, - w_retval) - return w_retval - - -def unicode_from_encoded_object(space, w_obj, encoding, errors): - if encoding is None: - encoding = getdefaultencoding(space) - w_retval = decode_object(space, w_obj, encoding, errors) - if not isinstance(w_retval, W_UnicodeObject): - raise oefmt(space.w_TypeError, - "decoder did not return a str object (type '%T')", - w_retval) - return w_retval - + from pypy.module._codecs.interp_codecs import decode + return decode(space, w_obj, encoding, errors) def unicode_from_object(space, w_obj): if space.is_w(space.type(w_obj), space.w_unicode): @@ -1281,6 +1252,7 @@ if space.lookup(w_obj, "__str__") is not None: return space.str(w_obj) return space.repr(w_obj) + def ascii_from_object(space, w_obj): """Implements builtins.ascii()""" # repr is guaranteed to be unicode @@ -1292,7 +1264,7 @@ # this is a performance and bootstrapping hack encoding = getdefaultencoding(space) if encoding != 'ascii': - return unicode_from_encoded_object(space, w_bytes, encoding, "strict") + return decode_object(space, w_bytes, encoding, "strict") s = space.bytes_w(w_bytes) unicodehelper.check_ascii_or_raise(space, s) return W_UnicodeObject(s, len(s)) @@ -1897,7 +1869,7 @@ if not isinstance(w_unistr, W_UnicodeObject): raise oefmt(space.w_TypeError, "expected unicode, got '%T'", w_unistr) utf8 = space.utf8_w(w_unistr) - lgt = space.len_w(w_unistr) + lgt = space.len_w(w_unistr) result = StringBuilder(lgt) pos = 0 for uchr in rutf8.Utf8StringIterator(utf8): @@ -1920,7 +1892,7 @@ raise OperationError(space.w_UnicodeEncodeError, space.newtuple([w_encoding, w_unistr, w_start, w_end, - w_reason])) + w_reason])) result.append(c) pos += 1 return result.build() diff --git a/rpython/rlib/rsre/rsre_utf8.py b/rpython/rlib/rsre/rsre_utf8.py --- a/rpython/rlib/rsre/rsre_utf8.py +++ b/rpython/rlib/rsre/rsre_utf8.py @@ -40,17 +40,23 @@ prev_indirect = prev def next_n(self, position, n, end_position): - for i in range(n): + i = 0 + # avoid range(n) since n can be quite large + while i < n: if position >= end_position: raise EndOfString position = rutf8.next_codepoint_pos(self._utf8, position) + i += 1 return position def prev_n(self, position, n, start_position): - for i in range(n): + i = 0 + # avoid range(n) since n can be quite large + while i < n: if position <= start_position: raise EndOfString position = rutf8.prev_codepoint_pos(self._utf8, position) + i += 1 assert position >= 0 return position diff --git a/rpython/rlib/test/test_rawrefcount_boehm.py b/rpython/rlib/test/test_rawrefcount_boehm.py --- a/rpython/rlib/test/test_rawrefcount_boehm.py +++ b/rpython/rlib/test/test_rawrefcount_boehm.py @@ -111,7 +111,7 @@ pyobjs.append(varname) return varname - for op in draw(strategies.lists(operations, average_size=250)): + for op in draw(strategies.lists(operations)): if op == 'new_gcobj': new_gcobj() elif op == 'new_pyobj': From pypy.commits at gmail.com Fri Mar 15 04:34:58 2019 From: pypy.commits at gmail.com (mattip) Date: Fri, 15 Mar 2019 01:34:58 -0700 (PDT) Subject: [pypy-commit] pypy default: update release note Message-ID: <5c8b63b2.1c69fb81.8c5d3.dbb4@mx.google.com> Author: Matti Picus Branch: Changeset: r96319:8d40471a2203 Date: 2019-03-15 10:34 +0200 http://bitbucket.org/pypy/pypy/changeset/8d40471a2203/ Log: update release note diff --git a/pypy/doc/release-v7.1.0.rst b/pypy/doc/release-v7.1.0.rst --- a/pypy/doc/release-v7.1.0.rst +++ b/pypy/doc/release-v7.1.0.rst @@ -24,6 +24,10 @@ Until we can work with downstream providers to distribute builds with PyPy, we have made packages for some common packages `available as wheels`_. +The `CFFI`_ backend has been updated to version 1.12.2. We reccomend using CFFI +rather than c-extensions to interact with C, and `cppyy`_ for interacting with +C++ code. + As always, this release is 100% compatible with the previous one and fixed several issues and bugs raised by the growing community of PyPy users. We strongly recommend updating. @@ -31,7 +35,7 @@ The PyPy3.6 release is still not production quality so your mileage may vary. There are open issues with incomplete compatibility and c-extension support. -You can download the v7.0 releases here: +You can download the v7.1 releases here: http://pypy.org/download.html @@ -47,7 +51,7 @@ .. _`PyPy`: index.html .. _`RPython`: https://rpython.readthedocs.org .. _`help`: project-ideas.html -.. _`cffi`: http://cffi.readthedocs.io +.. _`CFFI`: http://cffi.readthedocs.io .. _`cppyy`: https://cppyy.readthedocs.io .. _`available as wheels`: https://github.com/antocuni/pypy-wheels @@ -61,7 +65,7 @@ We also welcome developers of other `dynamic languages`_ to see what RPython can do for them. -The PyPy release supports: +This PyPy release supports: * **x86** machines on most common operating systems (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) @@ -71,7 +75,8 @@ * **s390x** running Linux Unfortunately at the moment of writing our ARM buildbots are out of service, -so for now we are **not** releasing any binary for the ARM architecture. +so for now we are **not** releasing any binary for the ARM architecture, +although PyPy does support ARM 32 bit processors. .. _`PyPy and CPython 2.7.x`: http://speed.pypy.org .. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html @@ -80,5 +85,37 @@ Changelog ========= -If not specified, the changes are shared across versions +Changes shared across versions +* Use utf8 internally to represent unicode, with the goal of never using + rpython-level unicode +* Update ``cffi`` to 1.12.2 +* Improve performance of ``long`` operations where one of the operands fits + into an ``int`` +* Since _ctypes is implemented in pure python over libffi, add interfaces and + methods to support the buffer interface from python. Specifically, add a + ``__pypy__.newmemoryview`` function to create a memoryview and extend the use + of the PyPy-specific ``__buffer__`` class method. This enables better + buffer sharing between ctypes and NumPy. +* Add copying to zlib +* Improve register allocation in the JIT by using better heuristics +* Include ```` on Gnu/Hurd +* Mostly for completeness sake: support for ``rlib.jit.promote_unicode``, which + behaves like ``promote_string``, but for rpython unicode objects +* Correctly initialize the ``d_type`` and ``d_name`` members of builtin + descriptors to fix a segfault related to classmethods in Cython +* Expand documentation of ``__pypy_`` module + +C-API (cpyext) improvements shared across versions + +* Move PyTuple_Type.tp_new to C +* Call internal methods from ``PyDict_XXXItem()`` instead of going through + dunder methods (CPython cpyext compatibility) + +Python 3.6 only + +* Support for os.PathLike in the posix module +* Update ``idellib`` for 3.6.1 +* Make ``BUILD_CONST_KEY_MAP`` JIT-friendly +* Adapt code that optimizes ``sys.exc_info()`` to wordcode +* Fix annotation bug found by ``attrs`` From pypy.commits at gmail.com Fri Mar 15 04:54:24 2019 From: pypy.commits at gmail.com (arigo) Date: Fri, 15 Mar 2019 01:54:24 -0700 (PDT) Subject: [pypy-commit] pypy default: Mention ordered dict Message-ID: <5c8b6840.1c69fb81.167f9.9dc8@mx.google.com> Author: Armin Rigo Branch: Changeset: r96320:4a9d9c5402d3 Date: 2019-03-15 09:54 +0100 http://bitbucket.org/pypy/pypy/changeset/4a9d9c5402d3/ Log: Mention ordered dict diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -482,6 +482,9 @@ * SyntaxError_ s try harder to give details about the cause of the failure, so the error messages are not the same as in CPython +* Dictionaries and sets are ordered on PyPy. On CPython < 3.6 they are not; + on CPython >= 3.6 dictionaries (but not sets) are ordered. + .. _extension-modules: From pypy.commits at gmail.com Fri Mar 15 06:22:45 2019 From: pypy.commits at gmail.com (mattip) Date: Fri, 15 Mar 2019 03:22:45 -0700 (PDT) Subject: [pypy-commit] pypy default: Added tag release-pypy2.7-v7.1.0 for changeset 990cef41fe11 Message-ID: <5c8b7cf5.1c69fb81.a926b.1bca@mx.google.com> Author: Matti Picus Branch: Changeset: r96321:c8f95abe6268 Date: 2019-03-15 12:21 +0200 http://bitbucket.org/pypy/pypy/changeset/c8f95abe6268/ Log: Added tag release-pypy2.7-v7.1.0 for changeset 990cef41fe11 diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -67,3 +67,4 @@ 928a4f70d3de7d17449456946154c5da6e600162 release-pypy3.5-v7.0.0 dab365a465140aa79a5f3ba4db784c4af4d5c195 release-pypy3.6-v7.0.0 fb40f7a5524c77b80e6c468e087d621610137261 release-pypy3.6-v7.0.0 +990cef41fe11e5d46b019a46aa956ff46ea1a234 release-pypy2.7-v7.1.0 From pypy.commits at gmail.com Fri Mar 15 06:22:47 2019 From: pypy.commits at gmail.com (mattip) Date: Fri, 15 Mar 2019 03:22:47 -0700 (PDT) Subject: [pypy-commit] pypy default: Added tag release-pypy3.6-v7.1.0 for changeset bb0d05b190b9 Message-ID: <5c8b7cf7.1c69fb81.37976.9598@mx.google.com> Author: Matti Picus Branch: Changeset: r96322:a9a50f3a32cd Date: 2019-03-15 12:21 +0200 http://bitbucket.org/pypy/pypy/changeset/a9a50f3a32cd/ Log: Added tag release-pypy3.6-v7.1.0 for changeset bb0d05b190b9 diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -68,3 +68,4 @@ dab365a465140aa79a5f3ba4db784c4af4d5c195 release-pypy3.6-v7.0.0 fb40f7a5524c77b80e6c468e087d621610137261 release-pypy3.6-v7.0.0 990cef41fe11e5d46b019a46aa956ff46ea1a234 release-pypy2.7-v7.1.0 +bb0d05b190b9c579f0c889a368636e14f6205bab release-pypy3.6-v7.1.0 From pypy.commits at gmail.com Fri Mar 15 07:01:18 2019 From: pypy.commits at gmail.com (mattip) Date: Fri, 15 Mar 2019 04:01:18 -0700 (PDT) Subject: [pypy-commit] pypy.org extradoc: update download page for 7.1 sha256 checksums, do not regenerate yet Message-ID: <5c8b85fe.1c69fb81.d2e0b.5ac2@mx.google.com> Author: Matti Picus Branch: extradoc Changeset: r938:916dbc49d944 Date: 2019-03-15 13:00 +0200 http://bitbucket.org/pypy/pypy.org/changeset/916dbc49d944/ Log: update download page for 7.1 sha256 checksums, do not regenerate yet diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -13,17 +13,17 @@ performance improvements. We provide binaries for x86, ARM, PPC and s390x running on different operating systems such as -Linux, Mac OS X and Windows (`what's new in PyPy 7.0?`_): +Linux, Mac OS X and Windows (`what's new in PyPy 7.1?`_): -* the Python2.7 compatible release — **PyPy2.7 v7.0** +* the Python2.7 compatible release — **PyPy2.7 v7.1** * the Python3.5 compatible release — **PyPy3.5 v7.0** -* the Python3.6 compatible release, alpha quality — **PyPy3.6 v7.0** +* the Python3.6 compatible release, alpha quality — **PyPy3.6 v7.1** * the Python2.7 Software Transactional Memory special release — **PyPy-STM 2.5.1** (Linux x86-64 only) -.. _what's new in PyPy 7.0?: http://doc.pypy.org/en/latest/release-v7.0.0.html +.. _what's new in PyPy 7.1?: http://doc.pypy.org/en/latest/release-v7.1.0.html .. class:: download_menu @@ -80,7 +80,7 @@ .. _release: -Python2.7 compatible PyPy 7.0 +Python2.7 compatible PyPy 7.1 ------------------------------- .. class:: download_menu @@ -98,15 +98,15 @@ * `All our downloads,`__ including previous versions. We also have a mirror_, but please use only if you have troubles accessing the links above -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.0.0-linux32.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.0.0-linux64.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.0.0-osx64.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.0.0-win32.zip -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.0.0-ppc64.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.0.0-ppc64le.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.0.0-s390x.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.0.0-src.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.0.0-src.zip +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.0-linux32.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.0-linux64.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.0-osx64.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.0-win32.zip +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.0-ppc64.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.0-ppc64le.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.0-s390x.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.0-src.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.1.0-src.zip .. _`vcredist_x86.exe`: http://www.microsoft.com/en-us/download/details.aspx?id=5582 .. __: https://bitbucket.org/pypy/pypy/downloads .. _mirror: http://buildbot.pypy.org/mirror/ @@ -153,23 +153,25 @@ libraries: ...``. Unless you want to hack a lot, try out the `portable Linux binaries`_. -Python 3.6 compatible PyPy3.6 v7.0.0-alpha -------------------------------------------- +Python 3.6 compatible PyPy3.6 v7.1.0-beta +----------------------------------------- .. class:: download_menu * `Linux x86-64 binary (64bit, built on Ubuntu 12.04 - 16.04)`__ (see ``[1]`` below) +* `Linux x86 binary (32bit, built on Ubuntu 12.04 - 16.04)`__ (see ``[1]`` below) * `Mac OS X binary (64bit)`__ (High Sierra >= 10.13, not for Sierra and below) -* `Windows binary (32bit)`__ **BETA** +* `Windows binary (32bit)`__ * `Source (tar.bz2)`__; `Source (zip)`__. See below for more about the sources. * `All our downloads,`__ including previous versions. We also have a mirror_, but please use only if you have troubles accessing the links above -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.0.0-linux64.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.0.0-osx64.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.0.0-win32.zip -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.0.0-src.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.0.0-src.zip +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.0-linux64.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.0-linux32.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.0-osx64.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.0-win32.zip +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.0-src.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.0-src.zip .. __: https://bitbucket.org/pypy/pypy/downloads @@ -255,8 +257,6 @@ +++++++++++++++++ Installation works on any recent PyPy (the release_ above is fine). -The currently released numpy 1.13 works except for ``nditers`` with the -``updateifcopy`` flag. For example, without using a virtualenv:: $ ./pypy-xxx/bin/pypy -m ensurepip @@ -457,7 +457,26 @@ Checksums --------- -Here are the checksums for each of the downloads of PyPy 7.0.0 and the older 6.0.0. +Here are the checksums for each of the downloads of PyPy 7.1.0, 7.0.0 and 6.0.0. + +pypy2.7-7.1.0 sha256:: + + deb13950ead91668db64e3a375e1e3c5ff041fe5a6e12d16887ef98b7946b046 pypy2.7-v7.1.0-linux32.tar.bz2 + a3e467eef5e048484ac35faa46dec99818689735926c69376b3f0a59637b1593 pypy2.7-v7.1.0-linux64.tar.bz2 + 6a40d230730ca2dc898ceba1f4022130f36bb599235cde976adc51086eaf2378 pypy2.7-v7.1.0-osx64.tar.bz2 + 82ee6158c5485df6d50812095bb82c7490519d8160d07f3dec4dc21933c473f5 pypy2.7-v7.1.0-s390x.tar.bz2 + 84ce5bc2867b224e2516ef431d78c6908d0182bc89444f7c1ef707443763754f pypy2.7-v7.1.0-src.tar.bz2 + 609a3d07e66f78a8001215abdfb3bacbc2f7cfeda141ccf1cac45c5249678521 pypy2.7-v7.1.0-src.zip + bb10276ff100c0208b11d6dfe95c53ea66fc90a1356a1094cb50ed1394b95eb7 pypy2.7-v7.1.0-win32.zip + +pypy 3.6-v7.1.0 sha256:: + + 315432517b09a5ec4afa91517f0af9e37c73557a4751a111ee88230a36061399 pypy3.6-v7.1.0-linux32.tar.bz2 + 580a3cc01517783aa24f4d5efe88b4323106d6952f9cf987adb11c909f11aa54 pypy3.6-v7.1.0-linux64.tar.bz2 + d68229c1755fa2d22fb1449be48a6f8681c8def0a3c93dae2929b95a0d535ff5 pypy3.6-v7.1.0-osx64.tar.bz2 + 181cd333b5aafa4bf041a1b249113ab493fb63d1880504e5912109d9aab74689 pypy3.6-v7.1.0-src.tar.bz2 + d2ba6f2d5a0e5a20edf525488912b86a92f296347481ffaa28b7a293ad48a4cd pypy3.6-v7.1.0-src.zip + 2b55b7e03e415790e19a45b8b63d86be20c9d4e1297b84e624d6bc275680d0fa pypy3.6-v7.1.0-win32.zip pypy2.7-7.0.0 sha256:: From pypy.commits at gmail.com Fri Mar 15 07:03:30 2019 From: pypy.commits at gmail.com (mattip) Date: Fri, 15 Mar 2019 04:03:30 -0700 (PDT) Subject: [pypy-commit] pypy default: typo (squeaky_pl) Message-ID: <5c8b8682.1c69fb81.69dd.bab4@mx.google.com> Author: Matti Picus Branch: Changeset: r96324:2265117e990a Date: 2019-03-15 13:02 +0200 http://bitbucket.org/pypy/pypy/changeset/2265117e990a/ Log: typo (squeaky_pl) diff --git a/pypy/doc/release-v7.1.0.rst b/pypy/doc/release-v7.1.0.rst --- a/pypy/doc/release-v7.1.0.rst +++ b/pypy/doc/release-v7.1.0.rst @@ -24,7 +24,7 @@ Until we can work with downstream providers to distribute builds with PyPy, we have made packages for some common packages `available as wheels`_. -The `CFFI`_ backend has been updated to version 1.12.2. We reccomend using CFFI +The `CFFI`_ backend has been updated to version 1.12.2. We recommend using CFFI rather than c-extensions to interact with C, and `cppyy`_ for interacting with C++ code. From pypy.commits at gmail.com Fri Mar 15 07:30:09 2019 From: pypy.commits at gmail.com (mattip) Date: Fri, 15 Mar 2019 04:30:09 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: try to debug s390x failure to build _blake2 Message-ID: <5c8b8cc1.1c69fb81.4bcf9.2d23@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r96325:4070744c27b1 Date: 2019-03-15 13:29 +0200 http://bitbucket.org/pypy/pypy/changeset/4070744c27b1/ Log: try to debug s390x failure to build _blake2 diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py --- a/pypy/tool/build_cffi_imports.py +++ b/pypy/tool/build_cffi_imports.py @@ -224,7 +224,7 @@ print("stdout:") print(stdout.decode('utf-8')) print("stderr:") - print(stderr.decode('utf-8')) + print(stderr) except: import traceback;traceback.print_exc() failures.append((key, module)) From pypy.commits at gmail.com Fri Mar 15 11:38:52 2019 From: pypy.commits at gmail.com (mattip) Date: Fri, 15 Mar 2019 08:38:52 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy3.6-v7.x: fix version to beta Message-ID: <5c8bc70c.1c69fb81.6d3b.3601@mx.google.com> Author: Matti Picus Branch: release-pypy3.6-v7.x Changeset: r96326:6fd188f8f903 Date: 2019-03-15 17:24 +0200 http://bitbucket.org/pypy/pypy/changeset/6fd188f8f903/ Log: fix version to beta diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -32,8 +32,8 @@ * module/sys/version.py * doc/conf.py */ -#define PYPY_VERSION "7.1.0" -#define PYPY_VERSION_NUM 0x07010000 +#define PYPY_VERSION "7.2.0-beta0" +#define PYPY_VERSION_NUM 0x07020000 /* Defined to mean a PyPy where cpyext holds more regular references to PyObjects, e.g. staying alive as long as the internal PyPy object stays alive. */ diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -13,7 +13,7 @@ # make sure to keep PYPY_VERSION in sync with: # module/cpyext/include/patchlevel.h # doc/conf.py -PYPY_VERSION = (7, 1, 0, "final", 0) +PYPY_VERSION = (7, 2, 0, "beta", 0) import pypy From pypy.commits at gmail.com Fri Mar 15 11:40:56 2019 From: pypy.commits at gmail.com (mattip) Date: Fri, 15 Mar 2019 08:40:56 -0700 (PDT) Subject: [pypy-commit] pypy default: Moved tag release-pypy3.6-v7.1.0 to changeset 6fd188f8f903 (from changeset bb0d05b190b9) Message-ID: <5c8bc788.1c69fb81.b950e.12fa@mx.google.com> Author: Matti Picus Branch: Changeset: r96327:3e943db40217 Date: 2019-03-15 17:39 +0200 http://bitbucket.org/pypy/pypy/changeset/3e943db40217/ Log: Moved tag release-pypy3.6-v7.1.0 to changeset 6fd188f8f903 (from changeset bb0d05b190b9) diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -69,3 +69,5 @@ fb40f7a5524c77b80e6c468e087d621610137261 release-pypy3.6-v7.0.0 990cef41fe11e5d46b019a46aa956ff46ea1a234 release-pypy2.7-v7.1.0 bb0d05b190b9c579f0c889a368636e14f6205bab release-pypy3.6-v7.1.0 +bb0d05b190b9c579f0c889a368636e14f6205bab release-pypy3.6-v7.1.0 +6fd188f8f903b7555920adf7d5e7fe21db1bd593 release-pypy3.6-v7.1.0 From pypy.commits at gmail.com Sat Mar 16 03:01:47 2019 From: pypy.commits at gmail.com (arigo) Date: Sat, 16 Mar 2019 00:01:47 -0700 (PDT) Subject: [pypy-commit] cffi default: py3.8 fixes Message-ID: <5c8c9f5b.1c69fb81.118f3.bcdd@mx.google.com> Author: Armin Rigo Branch: Changeset: r3243:272b53198be9 Date: 2019-03-16 08:00 +0100 http://bitbucket.org/cffi/cffi/changeset/272b53198be9/ Log: py3.8 fixes diff --git a/cffi/_embedding.h b/cffi/_embedding.h --- a/cffi/_embedding.h +++ b/cffi/_embedding.h @@ -169,8 +169,10 @@ global_dict = PyDict_New(); if (global_dict == NULL) goto error; - if (PyDict_SetItemString(global_dict, "__builtins__", - PyThreadState_GET()->interp->builtins) < 0) + PyObject *builtins = PyEval_GetBuiltins(); + if (builtins == NULL) + goto error; + if (PyDict_SetItemString(global_dict, "__builtins__", builtins) < 0) goto error; x = PyEval_EvalCode( #if PY_MAJOR_VERSION < 3 @@ -263,23 +265,33 @@ So we use a global variable as a simple spin lock. This global variable must be from 'libpythonX.Y.so', not from this cffi-based extension module, because it must be shared from - different cffi-based extension modules. We choose + different cffi-based extension modules. + + In Python < 3.8, we choose _PyParser_TokenNames[0] as a completely arbitrary pointer value that is never written to. The default is to point to the string "ENDMARKER". We change it temporarily to point to the next character in that string. (Yes, I know it's REALLY obscure.) + + In Python >= 3.8, this string array is no longer writable, so + instead we pick PyCapsuleType.tp_version_tag. We can't change + Python < 3.8 because someone might use a mixture of cffi + embedded modules, some of which were compiled before this file + changed. */ #ifdef WITH_THREAD +# if PY_VERSION_HEX < 0x03080000 char *volatile *lock = (char *volatile *)_PyParser_TokenNames; - char *old_value; + char *old_value, *locked_value; while (1) { /* spin loop */ old_value = *lock; + locked_value = old_value + 1; if (old_value[0] == 'E') { assert(old_value[1] == 'N'); - if (cffi_compare_and_swap(lock, old_value, old_value + 1)) + if (cffi_compare_and_swap(lock, old_value, locked_value)) break; } else { @@ -290,6 +302,27 @@ this is only run at start-up anyway. */ } } +# else + int volatile *lock = (int volatile *)&PyCapsule_Type.tp_version_tag; + int old_value, locked_value; + assert(!(PyCapsule_Type.tp_flags & Py_TPFLAGS_HAVE_VERSION_TAG)); + + while (1) { /* spin loop */ + old_value = *lock; + locked_value = -42; + if (old_value == 0) { + if (cffi_compare_and_swap(lock, old_value, locked_value)) + break; + } + else { + assert(old_value == locked_value); + /* should ideally do a spin loop instruction here, but + hard to do it portably and doesn't really matter I + think: PyEval_InitThreads() should be very fast, and + this is only run at start-up anyway. */ + } + } +# endif #endif /* call Py_InitializeEx() */ @@ -306,7 +339,7 @@ #ifdef WITH_THREAD /* release the lock */ - while (!cffi_compare_and_swap(lock, old_value + 1, old_value)) + while (!cffi_compare_and_swap(lock, locked_value, old_value)) ; #endif diff --git a/testing/cffi0/test_verify.py b/testing/cffi0/test_verify.py --- a/testing/cffi0/test_verify.py +++ b/testing/cffi0/test_verify.py @@ -20,7 +20,8 @@ extra_compile_args.append('-Qunused-arguments') else: # assume a standard gcc - extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion'] + extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion', + '-Wno-unused-parameter'] class FFI(FFI): def verify(self, *args, **kwds): diff --git a/testing/cffi1/test_verify1.py b/testing/cffi1/test_verify1.py --- a/testing/cffi1/test_verify1.py +++ b/testing/cffi1/test_verify1.py @@ -22,7 +22,8 @@ extra_compile_args.append('-Qunused-arguments') else: # assume a standard gcc - extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion'] + extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion', + '-Wno-unused-parameter'] class FFI(FFI): error = _cffi_backend.FFI.error diff --git a/testing/embedding/test_basic.py b/testing/embedding/test_basic.py --- a/testing/embedding/test_basic.py +++ b/testing/embedding/test_basic.py @@ -172,7 +172,8 @@ result = popen.stdout.read() err = popen.wait() if err: - raise OSError("%r failed with exit code %r" % (name, err)) + raise OSError("%r failed with exit code %r" % ( + os.path.join(path, executable_name), err)) return result From pypy.commits at gmail.com Sat Mar 16 03:15:14 2019 From: pypy.commits at gmail.com (arigo) Date: Sat, 16 Mar 2019 00:15:14 -0700 (PDT) Subject: [pypy-commit] cffi py3.8-interp-dict: Testing whether interp->dict seems to work. Message-ID: <5c8ca282.1c69fb81.5049b.3144@mx.google.com> Author: Armin Rigo Branch: py3.8-interp-dict Changeset: r3244:25b7ab34a165 Date: 2019-03-16 06:37 +0100 http://bitbucket.org/cffi/cffi/changeset/25b7ab34a165/ Log: Testing whether interp->dict seems to work. It's a new addition to Python 3.8: https://bugs.python.org/issue36124 From pypy.commits at gmail.com Sat Mar 16 03:15:16 2019 From: pypy.commits at gmail.com (arigo) Date: Sat, 16 Mar 2019 00:15:16 -0700 (PDT) Subject: [pypy-commit] cffi py3.8-interp-dict: attempt to use PyInterpreterState_GetDict() Message-ID: <5c8ca284.1c69fb81.4f3c5.712e@mx.google.com> Author: Armin Rigo Branch: py3.8-interp-dict Changeset: r3245:113fd85cc556 Date: 2019-03-16 08:03 +0100 http://bitbucket.org/cffi/cffi/changeset/113fd85cc556/ Log: attempt to use PyInterpreterState_GetDict() diff --git a/c/call_python.c b/c/call_python.c --- a/c/call_python.c +++ b/c/call_python.c @@ -1,10 +1,18 @@ #if PY_VERSION_HEX >= 0x03080000 -# define Py_BUILD_CORE -/* for access to the fields of PyInterpreterState */ -# include "internal/pycore_pystate.h" -# undef Py_BUILD_CORE +# define HAVE_PYINTERPSTATE_GETDICT #endif + +static PyObject *_current_interp_key(void) +{ + PyInterpreterState *interp = PyThreadState_GET()->interp; +#ifdef HAVE_PYINTERPSTATE_GETDICT + return PyInterpreterState_GetDict(interp); /* shared reference */ +#else + return interp->modules; +#endif +} + static PyObject *_get_interpstate_dict(void) { /* Hack around to return a dict that is subinterpreter-local. @@ -14,7 +22,7 @@ */ static PyObject *attr_name = NULL; PyThreadState *tstate; - PyObject *d, *builtins; + PyObject *d, *interpdict; int err; tstate = PyThreadState_GET(); @@ -23,8 +31,13 @@ return NULL; } - builtins = tstate->interp->builtins; - if (builtins == NULL) { + PyInterpreterState *interp = tstate->interp; +#ifdef HAVE_PYINTERPSTATE_GETDICT + interpdict = PyInterpreterState_GetDict(interp); /* shared reference */ +#else + interpdict = interp->builtins; +#endif + if (interpdict == NULL) { /* subinterpreter was cleared already, or is being cleared right now, to a point that is too much for us to continue */ return NULL; @@ -38,13 +51,13 @@ goto error; } - d = PyDict_GetItem(builtins, attr_name); + d = PyDict_GetItem(interpdict, attr_name); if (d == NULL) { d = PyDict_New(); if (d == NULL) goto error; - err = PyDict_SetItem(builtins, attr_name, d); - Py_DECREF(d); /* if successful, there is one ref left in builtins */ + err = PyDict_SetItem(interpdict, attr_name, d); + Py_DECREF(d); /* if successful, there is one ref left in interpdict */ if (err < 0) goto error; } @@ -163,7 +176,7 @@ if (infotuple == NULL) return 3; /* no ffi.def_extern() from this subinterpreter */ - new1 = PyThreadState_GET()->interp->modules; + new1 = _current_interp_key(); Py_INCREF(new1); Py_INCREF(infotuple); old1 = (PyObject *)externpy->reserved1; @@ -252,7 +265,7 @@ } else { PyGILState_STATE state = gil_ensure(); - if (externpy->reserved1 != PyThreadState_GET()->interp->modules) { + if (externpy->reserved1 != _current_interp_key()) { /* Update the (reserved1, reserved2) cache. This will fail if we didn't call @ffi.def_extern() in this particular subinterpreter. */ From pypy.commits at gmail.com Sat Mar 16 03:15:18 2019 From: pypy.commits at gmail.com (arigo) Date: Sat, 16 Mar 2019 00:15:18 -0700 (PDT) Subject: [pypy-commit] cffi py3.8-interp-dict: hg merge default Message-ID: <5c8ca286.1c69fb81.a926b.2bea@mx.google.com> Author: Armin Rigo Branch: py3.8-interp-dict Changeset: r3246:1f45194ae650 Date: 2019-03-16 08:03 +0100 http://bitbucket.org/cffi/cffi/changeset/1f45194ae650/ Log: hg merge default diff --git a/cffi/_embedding.h b/cffi/_embedding.h --- a/cffi/_embedding.h +++ b/cffi/_embedding.h @@ -169,8 +169,10 @@ global_dict = PyDict_New(); if (global_dict == NULL) goto error; - if (PyDict_SetItemString(global_dict, "__builtins__", - PyThreadState_GET()->interp->builtins) < 0) + PyObject *builtins = PyEval_GetBuiltins(); + if (builtins == NULL) + goto error; + if (PyDict_SetItemString(global_dict, "__builtins__", builtins) < 0) goto error; x = PyEval_EvalCode( #if PY_MAJOR_VERSION < 3 @@ -263,23 +265,33 @@ So we use a global variable as a simple spin lock. This global variable must be from 'libpythonX.Y.so', not from this cffi-based extension module, because it must be shared from - different cffi-based extension modules. We choose + different cffi-based extension modules. + + In Python < 3.8, we choose _PyParser_TokenNames[0] as a completely arbitrary pointer value that is never written to. The default is to point to the string "ENDMARKER". We change it temporarily to point to the next character in that string. (Yes, I know it's REALLY obscure.) + + In Python >= 3.8, this string array is no longer writable, so + instead we pick PyCapsuleType.tp_version_tag. We can't change + Python < 3.8 because someone might use a mixture of cffi + embedded modules, some of which were compiled before this file + changed. */ #ifdef WITH_THREAD +# if PY_VERSION_HEX < 0x03080000 char *volatile *lock = (char *volatile *)_PyParser_TokenNames; - char *old_value; + char *old_value, *locked_value; while (1) { /* spin loop */ old_value = *lock; + locked_value = old_value + 1; if (old_value[0] == 'E') { assert(old_value[1] == 'N'); - if (cffi_compare_and_swap(lock, old_value, old_value + 1)) + if (cffi_compare_and_swap(lock, old_value, locked_value)) break; } else { @@ -290,6 +302,27 @@ this is only run at start-up anyway. */ } } +# else + int volatile *lock = (int volatile *)&PyCapsule_Type.tp_version_tag; + int old_value, locked_value; + assert(!(PyCapsule_Type.tp_flags & Py_TPFLAGS_HAVE_VERSION_TAG)); + + while (1) { /* spin loop */ + old_value = *lock; + locked_value = -42; + if (old_value == 0) { + if (cffi_compare_and_swap(lock, old_value, locked_value)) + break; + } + else { + assert(old_value == locked_value); + /* should ideally do a spin loop instruction here, but + hard to do it portably and doesn't really matter I + think: PyEval_InitThreads() should be very fast, and + this is only run at start-up anyway. */ + } + } +# endif #endif /* call Py_InitializeEx() */ @@ -306,7 +339,7 @@ #ifdef WITH_THREAD /* release the lock */ - while (!cffi_compare_and_swap(lock, old_value + 1, old_value)) + while (!cffi_compare_and_swap(lock, locked_value, old_value)) ; #endif diff --git a/testing/cffi0/test_verify.py b/testing/cffi0/test_verify.py --- a/testing/cffi0/test_verify.py +++ b/testing/cffi0/test_verify.py @@ -20,7 +20,8 @@ extra_compile_args.append('-Qunused-arguments') else: # assume a standard gcc - extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion'] + extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion', + '-Wno-unused-parameter'] class FFI(FFI): def verify(self, *args, **kwds): diff --git a/testing/cffi1/test_verify1.py b/testing/cffi1/test_verify1.py --- a/testing/cffi1/test_verify1.py +++ b/testing/cffi1/test_verify1.py @@ -22,7 +22,8 @@ extra_compile_args.append('-Qunused-arguments') else: # assume a standard gcc - extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion'] + extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion', + '-Wno-unused-parameter'] class FFI(FFI): error = _cffi_backend.FFI.error diff --git a/testing/embedding/test_basic.py b/testing/embedding/test_basic.py --- a/testing/embedding/test_basic.py +++ b/testing/embedding/test_basic.py @@ -172,7 +172,8 @@ result = popen.stdout.read() err = popen.wait() if err: - raise OSError("%r failed with exit code %r" % (name, err)) + raise OSError("%r failed with exit code %r" % ( + os.path.join(path, executable_name), err)) return result From pypy.commits at gmail.com Sat Mar 16 13:16:42 2019 From: pypy.commits at gmail.com (andrewjlawrence) Date: Sat, 16 Mar 2019 10:16:42 -0700 (PDT) Subject: [pypy-commit] pypy winoverlapped: Fixed remaining tests in test_windows_events.py. Message-ID: <5c8d2f7a.1c69fb81.2d530.259c@mx.google.com> Author: andrewjlawrence Branch: winoverlapped Changeset: r96328:c0aff3718043 Date: 2019-03-16 17:14 +0000 http://bitbucket.org/pypy/pypy/changeset/c0aff3718043/ Log: Fixed remaining tests in test_windows_events.py. Added the overlapped class back into _winapi which was wrongly removed from there. For some reason cpython has two copies of the overlapped class. diff --git a/lib_pypy/_overlapped.py b/lib_pypy/_overlapped.py --- a/lib_pypy/_overlapped.py +++ b/lib_pypy/_overlapped.py @@ -24,6 +24,7 @@ # # Error Codes # +ERROR_IO_PENDING = 997 ERROR_PIPE_BUSY = 231 ERROR_NETNAME_DELETED = 64 # @@ -68,13 +69,15 @@ self.write_buffer = None self.type = OverlappedType.TYPE_NONE - if event == INVALID_HANDLE_VALUE or not event: + if event == _int2handle(INVALID_HANDLE_VALUE) or not event: event = _kernel32.CreateEventW(NULL, True, False, NULL) if event == _winapi.NULL: raise _winapi._WinError() if event: self.overlapped[0].hEvent = event + else: + raise _winapi._WinError() if self.overlapped[0].hEvent == _ffi.NULL: raise _winapi._WinError() @@ -254,7 +257,7 @@ @property def address(self): - return _ffi.addressof(self.overlapped[0]) + return self.overlapped def SetEvent(handle): @@ -292,7 +295,7 @@ def GetQueuedCompletionStatus(completionport, milliseconds): numberofbytes = _ffi.new('DWORD[1]', [0]) - completionkey = _ffi.new('ULONG**', _ffi.NULL) + completionkey = _ffi.new('ULONG**') if completionport is None: raise _winapi._WinError() @@ -313,27 +316,32 @@ return None return SetFromWindowsErr(err) - return (err, numberofbytes, completionkey[0], _ffi.addressof(overlapped[0][0])) + return (err, numberofbytes, _handle2int(completionkey[0]), _ffi.addressof(overlapped[0][0])) - at _ffi.callback("void(void*, bool)") + at _ffi.callback("void(void*, int)") def post_to_queue_callback(lpparameter, timerorwaitfired): pdata = _ffi.cast("PostCallbackData *", lpparameter) - _kernel32.PostQueuedCompletionStatus(pdata.hCompletionPort, timerorwaitfired, _ffi.cast("ULONG_PTR",0), pdata.Overlapped) + ret = _kernel32.PostQueuedCompletionStatus(pdata.hCompletionPort, timerorwaitfired, _ffi.cast("ULONG_PTR",0), pdata.Overlapped) + result = False +# if not ret: +# err = _winapi._WinError() def RegisterWaitWithQueue(object, completionport, ovaddress, miliseconds): - data = _ffi.new('PostCallbackData[1]') - newwaitobject = _ffi.new("HANDLE[1]") + data = _ffi.new('PostCallbackData*') + newwaitobject = _ffi.new("HANDLE*") data[0].hCompletionPort = completionport - data[0].Overlapped = _ffi.new("OVERLAPPED *",ovaddress[0]) - success = _kernel32.RegisterWaitForSingleObject(newwaitobject, - object, - _ffi.cast("WAITORTIMERCALLBACK",post_to_queue_callback), - data, - miliseconds, - _kernel32.WT_EXECUTEINWAITTHREAD | _kernel32.WT_EXECUTEONLYONCE) + data[0].Overlapped = ovaddress + ret = _kernel32.RegisterWaitForSingleObject(newwaitobject, + object, + _ffi.cast("WAITORTIMERCALLBACK",post_to_queue_callback), + data, + miliseconds, + _kernel32.WT_EXECUTEINWAITTHREAD | _kernel32.WT_EXECUTEONLYONCE) + if not ret: + raise _winapi._WinError() - return newwaitobject + return _handle2int(newwaitobject[0]) def ConnectPipe(address): err = _winapi.ERROR_PIPE_BUSY @@ -352,14 +360,22 @@ return _handle2int(handle) -#def UnregisterWaitEx(handle, event): -# waithandle = _int2handle(handle) -# waitevent = _int2handle(event) -# -# ret = _kernel32.UnregisterWaitEx(handle, event) -# -# if not ret: -# raise _winapi.WinError() +def UnregisterWaitEx(handle, event): + waithandle = _int2handle(handle) + waitevent = _int2handle(event) + + ret = _kernel32.UnregisterWaitEx(waithandle, waitevent) + + if not ret: + raise _winapi._WinError() + +def UnregisterWait(handle): + handle = _int2handle(handle) + + ret = _kernel32.UnregisterWait(handle) + + if not ret: + raise _winapi._WinError() # In CPython this function converts a windows error into a python object # Not sure what we should do here. diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py --- a/lib_pypy/_pypy_winbase_build.py +++ b/lib_pypy/_pypy_winbase_build.py @@ -140,6 +140,7 @@ BOOL WINAPI RegisterWaitForSingleObject(PHANDLE, HANDLE, WAITORTIMERCALLBACK, PVOID, ULONG, ULONG); BOOL WINAPI PostQueuedCompletionStatus(HANDLE, DWORD, ULONG_PTR, LPOVERLAPPED); BOOL WINAPI UnregisterWaitEx(HANDLE, HANDLE); +BOOL WINAPI UnregisterWait(HANDLE); BOOL WINAPI GetQueuedCompletionStatus(HANDLE, LPDWORD, ULONG**, LPOVERLAPPED*, DWORD); HANDLE WINAPI CreateIoCompletionPort(HANDLE, HANDLE, ULONG_PTR, DWORD); diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py --- a/lib_pypy/_pypy_winbase_cffi.py +++ b/lib_pypy/_pypy_winbase_cffi.py @@ -4,7 +4,7 @@ ffi = _cffi_backend.FFI('_pypy_winbase_cffi', _version = 0x2601, _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\xFB\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x13\x03\x00\x00\xEE\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\xF5\x03\x00\x00\x1A\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x1A\x11\x00\x00\x1A\x11\x00\x00\xF3\x03\x00\x00\xEF\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x03\x00\x00\x26\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x26\x11\x00\x00\x11\x11\x00\x00\xEA\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x14\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\xF4\x03\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\xE5\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x14\x03\x00\x00\x15\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x26\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\xFB\x03\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\xA1\x03\x00\x00\x7F\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x7F\x11\x00\x00\x7F\x11\x00\x00\x22\x11\x00\x00\x23\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x51\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x11\x11\x00\x00\x7F\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x02\x0F\x00\x00\x9C\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x9C\x0D\x00\x00\x00\x0F\x00\x00\x9C\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x00\xF2\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xF5\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xA4\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xA1\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xA7\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xA4\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xA7\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xA4\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x7F\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xA4\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xAD\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xA4\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\xFB\x0D\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x00\x0F\x00\x00\xFB\x0D\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x04\x09\x00\x00\x02\x09\x00\x00\xF1\x03\x00\x00\x05\x09\x00\x00\x06\x09\x00\x00\x03\x09\x00\x00\x07\x09\x00\x00\x02\x01\x00\x00\x40\x03\x00\x00\x01\x09\x00\x00\x00\x09\x00\x00\xFA\x03\x00\x00\x04\x01\x00\x00\x00\x01', - _globals = (b'\x00\x00\x33\x23CancelIo',0,b'\x00\x00\x36\x23CancelIoEx',0,b'\x00\x00\x33\x23CloseHandle',0,b'\x00\x00\x36\x23ConnectNamedPipe',0,b'\x00\x00\xA3\x23CreateEventA',0,b'\x00\x00\xA9\x23CreateEventW',0,b'\x00\x00\xAF\x23CreateFileA',0,b'\x00\x00\xDC\x23CreateFileW',0,b'\x00\x00\xCA\x23CreateIoCompletionPort',0,b'\x00\x00\xB8\x23CreateNamedPipeA',0,b'\x00\x00\xD2\x23CreateNamedPipeW',0,b'\x00\x00\x25\x23CreatePipe',0,b'\x00\x00\x19\x23CreateProcessA',0,b'\x00\x00\x7E\x23CreateProcessW',0,b'\x00\x00\x6E\x23DuplicateHandle',0,b'\x00\x00\xD0\x23GetCurrentProcess',0,b'\x00\x00\x53\x23GetExitCodeProcess',0,b'\x00\x00\x99\x23GetLastError',0,b'\x00\x00\x94\x23GetModuleFileNameW',0,b'\x00\x00\x3A\x23GetOverlappedResult',0,b'\x00\x00\x57\x23GetQueuedCompletionStatus',0,b'\x00\x00\xC7\x23GetStdHandle',0,b'\x00\x00\x99\x23GetVersion',0,b'\x00\x00\x64\x23PostQueuedCompletionStatus',0,b'\x00\x00\x10\x23ReadFile',0,b'\x00\x00\x2B\x23RegisterWaitForSingleObject',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\x8D\x23SetErrorMode',0,b'\x00\x00\x33\x23SetEvent',0,b'\x00\x00\x5E\x23SetNamedPipeHandleState',0,b'\x00\x00\x4F\x23TerminateProcess',0,b'\x00\x00\x6A\x23UnregisterWaitEx',0,b'\x00\x00\x46\x23WSARecv',0,b'\xFF\xFF\xFF\x1FWT_EXECUTEINWAITTHREAD',4,b'\xFF\xFF\xFF\x1FWT_EXECUTEONLYONCE',8,b'\x00\x00\x90\x23WaitForSingleObject',0,b'\x00\x00\x77\x23WriteFile',0,b'\x00\x00\x8A\x23_get_osfhandle',0,b'\x00\x00\x17\x23_getch',0,b'\x00\x00\x17\x23_getche',0,b'\x00\x00\x9E\x23_getwch',0,b'\x00\x00\x9E\x23_getwche',0,b'\x00\x00\x17\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\xA0\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x9B\x23_ungetwch',0,b'\x00\x00\xC2\x23socket',0), + _globals = (b'\x00\x00\x33\x23CancelIo',0,b'\x00\x00\x36\x23CancelIoEx',0,b'\x00\x00\x33\x23CloseHandle',0,b'\x00\x00\x36\x23ConnectNamedPipe',0,b'\x00\x00\xA3\x23CreateEventA',0,b'\x00\x00\xA9\x23CreateEventW',0,b'\x00\x00\xAF\x23CreateFileA',0,b'\x00\x00\xDC\x23CreateFileW',0,b'\x00\x00\xCA\x23CreateIoCompletionPort',0,b'\x00\x00\xB8\x23CreateNamedPipeA',0,b'\x00\x00\xD2\x23CreateNamedPipeW',0,b'\x00\x00\x25\x23CreatePipe',0,b'\x00\x00\x19\x23CreateProcessA',0,b'\x00\x00\x7E\x23CreateProcessW',0,b'\x00\x00\x6E\x23DuplicateHandle',0,b'\x00\x00\xD0\x23GetCurrentProcess',0,b'\x00\x00\x53\x23GetExitCodeProcess',0,b'\x00\x00\x99\x23GetLastError',0,b'\x00\x00\x94\x23GetModuleFileNameW',0,b'\x00\x00\x3A\x23GetOverlappedResult',0,b'\x00\x00\x57\x23GetQueuedCompletionStatus',0,b'\x00\x00\xC7\x23GetStdHandle',0,b'\x00\x00\x99\x23GetVersion',0,b'\x00\x00\x64\x23PostQueuedCompletionStatus',0,b'\x00\x00\x10\x23ReadFile',0,b'\x00\x00\x2B\x23RegisterWaitForSingleObject',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\x8D\x23SetErrorMode',0,b'\x00\x00\x33\x23SetEvent',0,b'\x00\x00\x5E\x23SetNamedPipeHandleState',0,b'\x00\x00\x4F\x23TerminateProcess',0,b'\x00\x00\x33\x23UnregisterWait',0,b'\x00\x00\x6A\x23UnregisterWaitEx',0,b'\x00\x00\x46\x23WSARecv',0,b'\xFF\xFF\xFF\x1FWT_EXECUTEINWAITTHREAD',4,b'\xFF\xFF\xFF\x1FWT_EXECUTEONLYONCE',8,b'\x00\x00\x90\x23WaitForSingleObject',0,b'\x00\x00\x77\x23WriteFile',0,b'\x00\x00\x8A\x23_get_osfhandle',0,b'\x00\x00\x17\x23_getch',0,b'\x00\x00\x17\x23_getche',0,b'\x00\x00\x9E\x23_getwch',0,b'\x00\x00\x9E\x23_getwche',0,b'\x00\x00\x17\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\xA0\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x9B\x23_ungetwch',0,b'\x00\x00\xC2\x23socket',0), _struct_unions = ((b'\x00\x00\x00\xF8\x00\x00\x00\x03$1',b'\x00\x00\xF7\x11DUMMYSTRUCTNAME',b'\x00\x00\x11\x11Pointer'),(b'\x00\x00\x00\xF7\x00\x00\x00\x02$2',b'\x00\x00\x13\x11Offset',b'\x00\x00\x13\x11OffsetHigh'),(b'\x00\x00\x00\xEF\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x11\x11hProcess',b'\x00\x00\x11\x11hThread',b'\x00\x00\x13\x11dwProcessId',b'\x00\x00\x13\x11dwThreadId'),(b'\x00\x00\x00\xF3\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x13\x11cb',b'\x00\x00\x1A\x11lpReserved',b'\x00\x00\x1A\x11lpDesktop',b'\x00\x00\x1A\x11lpTitle',b'\x00\x00\x13\x11dwX',b'\x00\x00\x13\x11dwY',b'\x00\x00\x13\x11dwXSize',b'\x00\x00\x13\x11dwYSize',b'\x00\x00\x13\x11dwXCountChars',b'\x00\x00\x13\x11dwYCountChars',b'\x00\x00\x13\x11dwFillAttribute',b'\x00\x00\x13\x11dwFlags',b'\x00\x00\x9C\x11wShowWindow',b'\x00\x00\x9C\x11cbReserved2',b'\x00\x00\xF9\x11lpReserved2',b'\x00\x00\x11\x11hStdInput',b'\x00\x00\x11\x11hStdOutput',b'\x00\x00\x11\x11hStdError'),(b'\x00\x00\x00\xEE\x00\x00\x00\x02_OVERLAPPED',b'\x00\x00\x13\x11Internal',b'\x00\x00\x13\x11InternalHigh',b'\x00\x00\xF8\x11DUMMYUNIONNAME',b'\x00\x00\x11\x11hEvent'),(b'\x00\x00\x00\xF1\x00\x00\x00\x02_PostCallbackData',b'\x00\x00\x11\x11hCompletionPort',b'\x00\x00\x15\x11Overlapped'),(b'\x00\x00\x00\xF2\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x13\x11nLength',b'\x00\x00\x11\x11lpSecurityDescriptor',b'\x00\x00\x01\x11bInheritHandle'),(b'\x00\x00\x00\xF4\x00\x00\x00\x02_WSABUF',b'\x00\x00\x13\x11len',b'\x00\x00\x1A\x11buf')), _typenames = (b'\x00\x00\x00\xF6LPFN_DISCONNECTEX',b'\x00\x00\x00\x15LPOVERLAPPED',b'\x00\x00\x00\x4DLPOVERLAPPED_COMPLETION_ROUTINE',b'\x00\x00\x00\x23LPPROCESS_INFORMATION',b'\x00\x00\x00\xF0LPPostCallbackData',b'\x00\x00\x00\xA4LPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x22LPSTARTUPINFO',b'\x00\x00\x00\x48LPWSABUF',b'\x00\x00\x00\xEEOVERLAPPED',b'\x00\x00\x00\xEFPROCESS_INFORMATION',b'\x00\x00\x00\xA4PSECURITY_ATTRIBUTES',b'\x00\x00\x00\xF1PostCallbackData',b'\x00\x00\x00\xF2SECURITY_ATTRIBUTES',b'\x00\x00\x00\x11SOCKET',b'\x00\x00\x00\xF3STARTUPINFO',b'\x00\x00\x00\x2EWAITORTIMERCALLBACK',b'\x00\x00\x00\xF4WSABUF',b'\x00\x00\x00\x9Cwint_t'), ) diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py --- a/lib_pypy/_winapi.py +++ b/lib_pypy/_winapi.py @@ -68,6 +68,84 @@ if not ret: raise _WinError() +class Overlapped(object): + def __init__(self, handle): + self.overlapped = _ffi.new('OVERLAPPED[1]') + self.handle = handle + self.readbuffer = None + self.pending = 0 + self.completed = 0 + self.writebuffer = None + self.overlapped[0].hEvent = \ + _kernel32.CreateEventW(NULL, True, False, NULL) + + def __del__(self): + # do this somehow else + xxx + err = _kernel32.GetLastError() + bytes = _ffi.new('DWORD[1]') + o = overlapped[0] + if overlapped[0].pending: + if _kernel32.CancelIoEx(o.handle, o.overlapped) & \ + self.GetOverlappedResult(o.handle, o.overlapped, _ffi.addressof(bytes), True): + # The operation is no longer pending, nothing to do + pass + else: + raise RuntimeError('deleting an overlapped struct with a pending operation not supported') + + @property + def event(self): + return None + + def GetOverlappedResult(self, wait): + transferred = _ffi.new('DWORD[1]', [0]) + res = _kernel32.GetOverlappedResult(self.handle, self.overlapped, transferred, wait != 0) + if res: + err = ERROR_SUCCESS + else: + err = GetLastError() + if err in (ERROR_SUCCESS, ERROR_MORE_DATA, ERROR_OPERATION_ABORTED): + self.completed = 1 + self.pending = 0 + elif res == ERROR_IO_INCOMPLETE: + pass + else: + self.pending = 0 + raise _WinError() + if self.completed and self.read_buffer: + if transferred != len(self.read_buffer): + raise _WinError() + return transferred[0], err + + def getbuffer(self): + xxx + return None + + def cancel(self): + xxx + return None + + +def ConnectNamedPipe(handle, overlapped=False): + if overlapped: + ov = Overlapped(handle) + else: + ov = Overlapped(None) + success = _kernel32.ConnectNamedPipe(handle, ov.overlapped) + if overlapped: + # Overlapped ConnectNamedPipe never returns a success code + assert success == 0 + err = _kernel32.GetLastError() + if err == ERROR_IO_PENDING: + ov.pending = 1 + elif err == ERROR_PIPE_CONNECTED: + _kernel32.SetEvent(ov.overlapped[0].hEvent) + else: + del ov + raise _WinError() + return ov + elif not success: + raise _WinError() def GetCurrentProcess(): return _handle2int(_kernel32.GetCurrentProcess()) From pypy.commits at gmail.com Sat Mar 16 14:30:24 2019 From: pypy.commits at gmail.com (andrewjlawrence) Date: Sat, 16 Mar 2019 11:30:24 -0700 (PDT) Subject: [pypy-commit] pypy winoverlapped: Set error on overlapped object and added wait for multiple objects. Message-ID: <5c8d40c0.1c69fb81.5ffbc.fcef@mx.google.com> Author: andrewjlawrence Branch: winoverlapped Changeset: r96329:3e255039d4dc Date: 2019-03-16 18:21 +0000 http://bitbucket.org/pypy/pypy/changeset/3e255039d4dc/ Log: Set error on overlapped object and added wait for multiple objects. diff --git a/lib_pypy/_overlapped.py b/lib_pypy/_overlapped.py --- a/lib_pypy/_overlapped.py +++ b/lib_pypy/_overlapped.py @@ -62,11 +62,12 @@ TYPE_TRANSMIT_FILE = 10 class Overlapped(object): - def __init__(self, event): + def __init__(self, event=_ffi.NULL): self.overlapped = _ffi.new('OVERLAPPED[1]') self.handle = _ffi.NULL self.read_buffer = None self.write_buffer = None + self.error = 0 self.type = OverlappedType.TYPE_NONE if event == _int2handle(INVALID_HANDLE_VALUE) or not event: @@ -93,6 +94,7 @@ err = _winapi.ERROR_SUCCESS if not ret: err = _kernel32.GetLastError() + self.error = err if err != _winapi.ERROR_SUCCESS and \ err != _winapi.ERROR_NOT_FOUND and \ err != _winapi.ERROR_OPERATION_ABORTED: @@ -118,10 +120,11 @@ err = _winapi.ERROR_SUCCESS else: err = _kernel32.GetLastError() + self.error = err if err != _winapi.ERROR_SUCCESS and err != _winapi.ERROR_MORE_DATA: if not (err == _winapi.ERROR_BROKEN_PIPE and (self.type == TYPE_READ or self.type == TYPE_READINTO)): - raise _winapi.WinError() + raise _winapi._WinError() if self.type == OverlappedType.TYPE_READ: return _ffi.unpack(self.read_buffer, transferred[0]) @@ -191,7 +194,8 @@ if success: err = _winapi.ERROR_SUCCESS else: - err = _kernel32.GetLastError() + err = _kernel32.GetLastError() + self.error = err if err == _winapi.ERROR_IO_PENDING | _winapi.ERROR_SUCCESS: return False @@ -214,6 +218,7 @@ err = _winapi.ERROR_SUCCESS else: err = _kernel32.GetLastError() + self.error = err if err == _winapi.ERROR_BROKEN_PIPE: mark_as_completed(self.overlapped) diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py --- a/lib_pypy/_pypy_winbase_build.py +++ b/lib_pypy/_pypy_winbase_build.py @@ -121,6 +121,7 @@ void *, BOOL, DWORD, wchar_t *, wchar_t *, LPSTARTUPINFO, LPPROCESS_INFORMATION); DWORD WINAPI WaitForSingleObject(HANDLE, DWORD); +DWORD WaitForMultipleObjects(DWORD, HANDLE*, BOOL, DWORD); BOOL WINAPI GetExitCodeProcess(HANDLE, LPDWORD); BOOL WINAPI TerminateProcess(HANDLE, UINT); HANDLE WINAPI GetStdHandle(DWORD); @@ -138,6 +139,7 @@ typedef VOID (WINAPI *WAITORTIMERCALLBACK) (PVOID, BOOL); BOOL WINAPI RegisterWaitForSingleObject(PHANDLE, HANDLE, WAITORTIMERCALLBACK, PVOID, ULONG, ULONG); + BOOL WINAPI PostQueuedCompletionStatus(HANDLE, DWORD, ULONG_PTR, LPOVERLAPPED); BOOL WINAPI UnregisterWaitEx(HANDLE, HANDLE); BOOL WINAPI UnregisterWait(HANDLE); diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py --- a/lib_pypy/_pypy_winbase_cffi.py +++ b/lib_pypy/_pypy_winbase_cffi.py @@ -3,8 +3,8 @@ ffi = _cffi_backend.FFI('_pypy_winbase_cffi', _version = 0x2601, - _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\xFB\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x13\x03\x00\x00\xEE\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\xF5\x03\x00\x00\x1A\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x1A\x11\x00\x00\x1A\x11\x00\x00\xF3\x03\x00\x00\xEF\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x03\x00\x00\x26\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x26\x11\x00\x00\x11\x11\x00\x00\xEA\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x14\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\xF4\x03\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\xE5\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x14\x03\x00\x00\x15\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x26\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\xFB\x03\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\xA1\x03\x00\x00\x7F\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x7F\x11\x00\x00\x7F\x11\x00\x00\x22\x11\x00\x00\x23\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x51\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x11\x11\x00\x00\x7F\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x02\x0F\x00\x00\x9C\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x9C\x0D\x00\x00\x00\x0F\x00\x00\x9C\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x00\xF2\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xF5\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xA4\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xA1\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xA7\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xA4\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xA7\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xA4\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x7F\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xA4\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xAD\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xA4\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\xFB\x0D\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x00\x0F\x00\x00\xFB\x0D\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x04\x09\x00\x00\x02\x09\x00\x00\xF1\x03\x00\x00\x05\x09\x00\x00\x06\x09\x00\x00\x03\x09\x00\x00\x07\x09\x00\x00\x02\x01\x00\x00\x40\x03\x00\x00\x01\x09\x00\x00\x00\x09\x00\x00\xFA\x03\x00\x00\x04\x01\x00\x00\x00\x01', - _globals = (b'\x00\x00\x33\x23CancelIo',0,b'\x00\x00\x36\x23CancelIoEx',0,b'\x00\x00\x33\x23CloseHandle',0,b'\x00\x00\x36\x23ConnectNamedPipe',0,b'\x00\x00\xA3\x23CreateEventA',0,b'\x00\x00\xA9\x23CreateEventW',0,b'\x00\x00\xAF\x23CreateFileA',0,b'\x00\x00\xDC\x23CreateFileW',0,b'\x00\x00\xCA\x23CreateIoCompletionPort',0,b'\x00\x00\xB8\x23CreateNamedPipeA',0,b'\x00\x00\xD2\x23CreateNamedPipeW',0,b'\x00\x00\x25\x23CreatePipe',0,b'\x00\x00\x19\x23CreateProcessA',0,b'\x00\x00\x7E\x23CreateProcessW',0,b'\x00\x00\x6E\x23DuplicateHandle',0,b'\x00\x00\xD0\x23GetCurrentProcess',0,b'\x00\x00\x53\x23GetExitCodeProcess',0,b'\x00\x00\x99\x23GetLastError',0,b'\x00\x00\x94\x23GetModuleFileNameW',0,b'\x00\x00\x3A\x23GetOverlappedResult',0,b'\x00\x00\x57\x23GetQueuedCompletionStatus',0,b'\x00\x00\xC7\x23GetStdHandle',0,b'\x00\x00\x99\x23GetVersion',0,b'\x00\x00\x64\x23PostQueuedCompletionStatus',0,b'\x00\x00\x10\x23ReadFile',0,b'\x00\x00\x2B\x23RegisterWaitForSingleObject',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\x8D\x23SetErrorMode',0,b'\x00\x00\x33\x23SetEvent',0,b'\x00\x00\x5E\x23SetNamedPipeHandleState',0,b'\x00\x00\x4F\x23TerminateProcess',0,b'\x00\x00\x33\x23UnregisterWait',0,b'\x00\x00\x6A\x23UnregisterWaitEx',0,b'\x00\x00\x46\x23WSARecv',0,b'\xFF\xFF\xFF\x1FWT_EXECUTEINWAITTHREAD',4,b'\xFF\xFF\xFF\x1FWT_EXECUTEONLYONCE',8,b'\x00\x00\x90\x23WaitForSingleObject',0,b'\x00\x00\x77\x23WriteFile',0,b'\x00\x00\x8A\x23_get_osfhandle',0,b'\x00\x00\x17\x23_getch',0,b'\x00\x00\x17\x23_getche',0,b'\x00\x00\x9E\x23_getwch',0,b'\x00\x00\x9E\x23_getwche',0,b'\x00\x00\x17\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\xA0\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x9B\x23_ungetwch',0,b'\x00\x00\xC2\x23socket',0), - _struct_unions = ((b'\x00\x00\x00\xF8\x00\x00\x00\x03$1',b'\x00\x00\xF7\x11DUMMYSTRUCTNAME',b'\x00\x00\x11\x11Pointer'),(b'\x00\x00\x00\xF7\x00\x00\x00\x02$2',b'\x00\x00\x13\x11Offset',b'\x00\x00\x13\x11OffsetHigh'),(b'\x00\x00\x00\xEF\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x11\x11hProcess',b'\x00\x00\x11\x11hThread',b'\x00\x00\x13\x11dwProcessId',b'\x00\x00\x13\x11dwThreadId'),(b'\x00\x00\x00\xF3\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x13\x11cb',b'\x00\x00\x1A\x11lpReserved',b'\x00\x00\x1A\x11lpDesktop',b'\x00\x00\x1A\x11lpTitle',b'\x00\x00\x13\x11dwX',b'\x00\x00\x13\x11dwY',b'\x00\x00\x13\x11dwXSize',b'\x00\x00\x13\x11dwYSize',b'\x00\x00\x13\x11dwXCountChars',b'\x00\x00\x13\x11dwYCountChars',b'\x00\x00\x13\x11dwFillAttribute',b'\x00\x00\x13\x11dwFlags',b'\x00\x00\x9C\x11wShowWindow',b'\x00\x00\x9C\x11cbReserved2',b'\x00\x00\xF9\x11lpReserved2',b'\x00\x00\x11\x11hStdInput',b'\x00\x00\x11\x11hStdOutput',b'\x00\x00\x11\x11hStdError'),(b'\x00\x00\x00\xEE\x00\x00\x00\x02_OVERLAPPED',b'\x00\x00\x13\x11Internal',b'\x00\x00\x13\x11InternalHigh',b'\x00\x00\xF8\x11DUMMYUNIONNAME',b'\x00\x00\x11\x11hEvent'),(b'\x00\x00\x00\xF1\x00\x00\x00\x02_PostCallbackData',b'\x00\x00\x11\x11hCompletionPort',b'\x00\x00\x15\x11Overlapped'),(b'\x00\x00\x00\xF2\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x13\x11nLength',b'\x00\x00\x11\x11lpSecurityDescriptor',b'\x00\x00\x01\x11bInheritHandle'),(b'\x00\x00\x00\xF4\x00\x00\x00\x02_WSABUF',b'\x00\x00\x13\x11len',b'\x00\x00\x1A\x11buf')), - _typenames = (b'\x00\x00\x00\xF6LPFN_DISCONNECTEX',b'\x00\x00\x00\x15LPOVERLAPPED',b'\x00\x00\x00\x4DLPOVERLAPPED_COMPLETION_ROUTINE',b'\x00\x00\x00\x23LPPROCESS_INFORMATION',b'\x00\x00\x00\xF0LPPostCallbackData',b'\x00\x00\x00\xA4LPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x22LPSTARTUPINFO',b'\x00\x00\x00\x48LPWSABUF',b'\x00\x00\x00\xEEOVERLAPPED',b'\x00\x00\x00\xEFPROCESS_INFORMATION',b'\x00\x00\x00\xA4PSECURITY_ATTRIBUTES',b'\x00\x00\x00\xF1PostCallbackData',b'\x00\x00\x00\xF2SECURITY_ATTRIBUTES',b'\x00\x00\x00\x11SOCKET',b'\x00\x00\x00\xF3STARTUPINFO',b'\x00\x00\x00\x2EWAITORTIMERCALLBACK',b'\x00\x00\x00\xF4WSABUF',b'\x00\x00\x00\x9Cwint_t'), + _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x01\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x13\x03\x00\x00\xF4\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\xFB\x03\x00\x00\x1A\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x1A\x11\x00\x00\x1A\x11\x00\x00\xF9\x03\x00\x00\xF5\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x03\x00\x00\x26\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x26\x11\x00\x00\x11\x11\x00\x00\xF0\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x14\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\xFA\x03\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\xEB\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x14\x03\x00\x00\x15\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x26\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x01\x03\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\xA7\x03\x00\x00\x7F\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x7F\x11\x00\x00\x7F\x11\x00\x00\x22\x11\x00\x00\x23\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x51\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x0A\x01\x00\x00\x26\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x13\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x11\x11\x00\x00\x7F\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x02\x0F\x00\x00\xA2\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\xA2\x0D\x00\x00\x00\x0F\x00\x00\xA2\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x00\xF8\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xFB\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xAA\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xA7\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xAD\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xAA\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xAD\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xAA\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x7F\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xAA\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xB3\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xAA\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x01\x01\x0D\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x00\x0F\x00\x01\x01\x0D\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x04\x09\x00\x00\x02\x09\x00\x00\xF7\x03\x00\x00\x05\x09\x00\x00\x06\x09\x00\x00\x03\x09\x00\x00\x07\x09\x00\x00\x02\x01\x00\x00\x40\x03\x00\x00\x01\x09\x00\x00\x00\x09\x00\x01\x00\x03\x00\x00\x04\x01\x00\x00\x00\x01', + _globals = (b'\x00\x00\x33\x23CancelIo',0,b'\x00\x00\x36\x23CancelIoEx',0,b'\x00\x00\x33\x23CloseHandle',0,b'\x00\x00\x36\x23ConnectNamedPipe',0,b'\x00\x00\xA9\x23CreateEventA',0,b'\x00\x00\xAF\x23CreateEventW',0,b'\x00\x00\xB5\x23CreateFileA',0,b'\x00\x00\xE2\x23CreateFileW',0,b'\x00\x00\xD0\x23CreateIoCompletionPort',0,b'\x00\x00\xBE\x23CreateNamedPipeA',0,b'\x00\x00\xD8\x23CreateNamedPipeW',0,b'\x00\x00\x25\x23CreatePipe',0,b'\x00\x00\x19\x23CreateProcessA',0,b'\x00\x00\x7E\x23CreateProcessW',0,b'\x00\x00\x6E\x23DuplicateHandle',0,b'\x00\x00\xD6\x23GetCurrentProcess',0,b'\x00\x00\x53\x23GetExitCodeProcess',0,b'\x00\x00\x9F\x23GetLastError',0,b'\x00\x00\x9A\x23GetModuleFileNameW',0,b'\x00\x00\x3A\x23GetOverlappedResult',0,b'\x00\x00\x57\x23GetQueuedCompletionStatus',0,b'\x00\x00\xCD\x23GetStdHandle',0,b'\x00\x00\x9F\x23GetVersion',0,b'\x00\x00\x64\x23PostQueuedCompletionStatus',0,b'\x00\x00\x10\x23ReadFile',0,b'\x00\x00\x2B\x23RegisterWaitForSingleObject',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\x8D\x23SetErrorMode',0,b'\x00\x00\x33\x23SetEvent',0,b'\x00\x00\x5E\x23SetNamedPipeHandleState',0,b'\x00\x00\x4F\x23TerminateProcess',0,b'\x00\x00\x33\x23UnregisterWait',0,b'\x00\x00\x6A\x23UnregisterWaitEx',0,b'\x00\x00\x46\x23WSARecv',0,b'\xFF\xFF\xFF\x1FWT_EXECUTEINWAITTHREAD',4,b'\xFF\xFF\xFF\x1FWT_EXECUTEONLYONCE',8,b'\x00\x00\x90\x23WaitForMultipleObjects',0,b'\x00\x00\x96\x23WaitForSingleObject',0,b'\x00\x00\x77\x23WriteFile',0,b'\x00\x00\x8A\x23_get_osfhandle',0,b'\x00\x00\x17\x23_getch',0,b'\x00\x00\x17\x23_getche',0,b'\x00\x00\xA4\x23_getwch',0,b'\x00\x00\xA4\x23_getwche',0,b'\x00\x00\x17\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\xA6\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\xA1\x23_ungetwch',0,b'\x00\x00\xC8\x23socket',0), + _struct_unions = ((b'\x00\x00\x00\xFE\x00\x00\x00\x03$1',b'\x00\x00\xFD\x11DUMMYSTRUCTNAME',b'\x00\x00\x11\x11Pointer'),(b'\x00\x00\x00\xFD\x00\x00\x00\x02$2',b'\x00\x00\x13\x11Offset',b'\x00\x00\x13\x11OffsetHigh'),(b'\x00\x00\x00\xF5\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x11\x11hProcess',b'\x00\x00\x11\x11hThread',b'\x00\x00\x13\x11dwProcessId',b'\x00\x00\x13\x11dwThreadId'),(b'\x00\x00\x00\xF9\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x13\x11cb',b'\x00\x00\x1A\x11lpReserved',b'\x00\x00\x1A\x11lpDesktop',b'\x00\x00\x1A\x11lpTitle',b'\x00\x00\x13\x11dwX',b'\x00\x00\x13\x11dwY',b'\x00\x00\x13\x11dwXSize',b'\x00\x00\x13\x11dwYSize',b'\x00\x00\x13\x11dwXCountChars',b'\x00\x00\x13\x11dwYCountChars',b'\x00\x00\x13\x11dwFillAttribute',b'\x00\x00\x13\x11dwFlags',b'\x00\x00\xA2\x11wShowWindow',b'\x00\x00\xA2\x11cbReserved2',b'\x00\x00\xFF\x11lpReserved2',b'\x00\x00\x11\x11hStdInput',b'\x00\x00\x11\x11hStdOutput',b'\x00\x00\x11\x11hStdError'),(b'\x00\x00\x00\xF4\x00\x00\x00\x02_OVERLAPPED',b'\x00\x00\x13\x11Internal',b'\x00\x00\x13\x11InternalHigh',b'\x00\x00\xFE\x11DUMMYUNIONNAME',b'\x00\x00\x11\x11hEvent'),(b'\x00\x00\x00\xF7\x00\x00\x00\x02_PostCallbackData',b'\x00\x00\x11\x11hCompletionPort',b'\x00\x00\x15\x11Overlapped'),(b'\x00\x00\x00\xF8\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x13\x11nLength',b'\x00\x00\x11\x11lpSecurityDescriptor',b'\x00\x00\x01\x11bInheritHandle'),(b'\x00\x00\x00\xFA\x00\x00\x00\x02_WSABUF',b'\x00\x00\x13\x11len',b'\x00\x00\x1A\x11buf')), + _typenames = (b'\x00\x00\x00\xFCLPFN_DISCONNECTEX',b'\x00\x00\x00\x15LPOVERLAPPED',b'\x00\x00\x00\x4DLPOVERLAPPED_COMPLETION_ROUTINE',b'\x00\x00\x00\x23LPPROCESS_INFORMATION',b'\x00\x00\x00\xF6LPPostCallbackData',b'\x00\x00\x00\xAALPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x22LPSTARTUPINFO',b'\x00\x00\x00\x48LPWSABUF',b'\x00\x00\x00\xF4OVERLAPPED',b'\x00\x00\x00\xF5PROCESS_INFORMATION',b'\x00\x00\x00\xAAPSECURITY_ATTRIBUTES',b'\x00\x00\x00\xF7PostCallbackData',b'\x00\x00\x00\xF8SECURITY_ATTRIBUTES',b'\x00\x00\x00\x11SOCKET',b'\x00\x00\x00\xF9STARTUPINFO',b'\x00\x00\x00\x2EWAITORTIMERCALLBACK',b'\x00\x00\x00\xFAWSABUF',b'\x00\x00\x00\xA2wint_t'), ) diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py --- a/lib_pypy/_winapi.py +++ b/lib_pypy/_winapi.py @@ -81,11 +81,10 @@ def __del__(self): # do this somehow else - xxx err = _kernel32.GetLastError() bytes = _ffi.new('DWORD[1]') - o = overlapped[0] - if overlapped[0].pending: + o = self.overlapped[0] + if self.pending: if _kernel32.CancelIoEx(o.handle, o.overlapped) & \ self.GetOverlappedResult(o.handle, o.overlapped, _ffi.addressof(bytes), True): # The operation is no longer pending, nothing to do @@ -112,8 +111,8 @@ else: self.pending = 0 raise _WinError() - if self.completed and self.read_buffer: - if transferred != len(self.read_buffer): + if self.completed and self.readbuffer: + if transferred != len(self.readbuffer): raise _WinError() return transferred[0], err @@ -127,6 +126,7 @@ def ConnectNamedPipe(handle, overlapped=False): + handle = _int2handle(handle) if overlapped: ov = Overlapped(handle) else: @@ -219,6 +219,18 @@ return res +def WaitForMultipleObjects(handle_sequence, waitflag, milliseconds): + if len(handle_sequence) > MAXIMUM_WAIT_OBJECTS: + return None + + # CPython makes the wait interruptible by ctrl-c. We need to add this in at some point + res = _kernel32.WaitForMultipleObjects(len(handle_sequence), handle_sequence, waitflag, milliseconds) + + if res == WAIT_FAILED: + raise _WinError() + return int(res) + + def GetExitCodeProcess(handle): # CPython: the first argument is expected to be an integer. code = _ffi.new("DWORD[1]") @@ -262,6 +274,7 @@ raise _WinError() return _ffi.string(buf) + # #define macros from WinBase.h and elsewhere STD_INPUT_HANDLE = -10 STD_OUTPUT_HANDLE = -11 @@ -274,6 +287,7 @@ WAIT_OBJECT_0 = 0 WAIT_ABANDONED_0 = 0x80 WAIT_TIMEOUT = 0x102 +WAIT_FAILED = 0xFFFFFFFF CREATE_NEW_CONSOLE = 0x010 CREATE_NEW_PROCESS_GROUP = 0x200 CREATE_UNICODE_ENVIRONMENT = 0x400 @@ -332,3 +346,4 @@ OPEN_ALWAYS = 4 TRUNCATE_EXISTING = 5 +MAXIMUM_WAIT_OBJECTS = 64 \ No newline at end of file From pypy.commits at gmail.com Sun Mar 17 03:17:43 2019 From: pypy.commits at gmail.com (andrewjlawrence) Date: Sun, 17 Mar 2019 00:17:43 -0700 (PDT) Subject: [pypy-commit] pypy winoverlapped: Added WSASend Message-ID: <5c8df497.1c69fb81.91d85.a8ad@mx.google.com> Author: andrewjlawrence Branch: winoverlapped Changeset: r96330:2e8a37df02ef Date: 2019-03-17 07:16 +0000 http://bitbucket.org/pypy/pypy/changeset/2e8a37df02ef/ Log: Added WSASend diff --git a/lib_pypy/_overlapped.py b/lib_pypy/_overlapped.py --- a/lib_pypy/_overlapped.py +++ b/lib_pypy/_overlapped.py @@ -27,6 +27,9 @@ ERROR_IO_PENDING = 997 ERROR_PIPE_BUSY = 231 ERROR_NETNAME_DELETED = 64 + +SOCKET_ERROR = -1 + # # Status Codes # @@ -167,7 +170,7 @@ wsabuff[0].buf = allocatedbuffer result = _winsock2.WSARecv(handle, wsabuff, _int2dword(1), nread, pflags, self.overlapped, _ffi.NULL) - if result < 0: + if result == SOCKET_ERROR: self.error = _kernel32.GetLastError() else: self.error = _winapi.ERROR_SUCCESS @@ -181,6 +184,34 @@ self.type = OverlappedType.TYPE_NOT_STARTED raise _winapi._WinError() + def WSASend(self ,handle, bufobj, flags): + handle = _int2handle(handle) + + if self.type != OverlappedType.TYPE_NONE: + raise _winapi._WinError() + self.write_buffer = bufobj + self.type = OverlappedType.TYPE_WRITE + self.handle = handle + + wsabuff = _ffi.new("WSABUF[1]") + wsabuff[0].len = len(bufobj) + wsabuff[0].buf = _ffi.new("CHAR[]", bufobj) + nwritten = _ffi.new("LPDWORD") + + + result = _winsock2.WSASend(handle, wsabuff, _int2dword(1), nwritten, flags, self.overlapped, _ffi.NULL) + + if result == SOCKET_ERROR: + self.error = _kernel32.GetLastError() + else: + self.error = _winapi.ERROR_SUCCESS + + if self.error in [_winapi.ERROR_SUCCESS, _winapi.ERROR_IO_PENDING]: + return None + else: + self.type = OverlappedType.TYPE_NOT_STARTED + raise _winapi._WinError() + def getresult(self, wait=False): return self.GetOverlappedResult(wait) diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py --- a/lib_pypy/_pypy_winbase_build.py +++ b/lib_pypy/_pypy_winbase_build.py @@ -168,7 +168,7 @@ typedef VOID (*LPOVERLAPPED_COMPLETION_ROUTINE) (DWORD, DWORD, LPVOID); int __stdcall WSARecv(SOCKET, LPWSABUF, DWORD, LPDWORD, LPDWORD, LPOVERLAPPED, LPOVERLAPPED_COMPLETION_ROUTINE); - +int __stdcall WSASend(SOCKET, LPWSABUF, DWORD, LPDWORD, DWORD, LPOVERLAPPED, LPOVERLAPPED_COMPLETION_ROUTINE); """) diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py --- a/lib_pypy/_pypy_winbase_cffi.py +++ b/lib_pypy/_pypy_winbase_cffi.py @@ -3,8 +3,8 @@ ffi = _cffi_backend.FFI('_pypy_winbase_cffi', _version = 0x2601, - _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x01\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x13\x03\x00\x00\xF4\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\xFB\x03\x00\x00\x1A\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x1A\x11\x00\x00\x1A\x11\x00\x00\xF9\x03\x00\x00\xF5\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x03\x00\x00\x26\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x26\x11\x00\x00\x11\x11\x00\x00\xF0\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x14\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\xFA\x03\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\xEB\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x14\x03\x00\x00\x15\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x26\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x01\x03\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\xA7\x03\x00\x00\x7F\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x7F\x11\x00\x00\x7F\x11\x00\x00\x22\x11\x00\x00\x23\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x51\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x0A\x01\x00\x00\x26\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x13\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x11\x11\x00\x00\x7F\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x02\x0F\x00\x00\xA2\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\xA2\x0D\x00\x00\x00\x0F\x00\x00\xA2\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x00\xF8\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xFB\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xAA\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xA7\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xAD\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xAA\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xAD\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xAA\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x7F\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xAA\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xB3\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xAA\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x01\x01\x0D\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x00\x0F\x00\x01\x01\x0D\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x04\x09\x00\x00\x02\x09\x00\x00\xF7\x03\x00\x00\x05\x09\x00\x00\x06\x09\x00\x00\x03\x09\x00\x00\x07\x09\x00\x00\x02\x01\x00\x00\x40\x03\x00\x00\x01\x09\x00\x00\x00\x09\x00\x01\x00\x03\x00\x00\x04\x01\x00\x00\x00\x01', - _globals = (b'\x00\x00\x33\x23CancelIo',0,b'\x00\x00\x36\x23CancelIoEx',0,b'\x00\x00\x33\x23CloseHandle',0,b'\x00\x00\x36\x23ConnectNamedPipe',0,b'\x00\x00\xA9\x23CreateEventA',0,b'\x00\x00\xAF\x23CreateEventW',0,b'\x00\x00\xB5\x23CreateFileA',0,b'\x00\x00\xE2\x23CreateFileW',0,b'\x00\x00\xD0\x23CreateIoCompletionPort',0,b'\x00\x00\xBE\x23CreateNamedPipeA',0,b'\x00\x00\xD8\x23CreateNamedPipeW',0,b'\x00\x00\x25\x23CreatePipe',0,b'\x00\x00\x19\x23CreateProcessA',0,b'\x00\x00\x7E\x23CreateProcessW',0,b'\x00\x00\x6E\x23DuplicateHandle',0,b'\x00\x00\xD6\x23GetCurrentProcess',0,b'\x00\x00\x53\x23GetExitCodeProcess',0,b'\x00\x00\x9F\x23GetLastError',0,b'\x00\x00\x9A\x23GetModuleFileNameW',0,b'\x00\x00\x3A\x23GetOverlappedResult',0,b'\x00\x00\x57\x23GetQueuedCompletionStatus',0,b'\x00\x00\xCD\x23GetStdHandle',0,b'\x00\x00\x9F\x23GetVersion',0,b'\x00\x00\x64\x23PostQueuedCompletionStatus',0,b'\x00\x00\x10\x23ReadFile',0,b'\x00\x00\x2B\x23RegisterWaitForSingleObject',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\x8D\x23SetErrorMode',0,b'\x00\x00\x33\x23SetEvent',0,b'\x00\x00\x5E\x23SetNamedPipeHandleState',0,b'\x00\x00\x4F\x23TerminateProcess',0,b'\x00\x00\x33\x23UnregisterWait',0,b'\x00\x00\x6A\x23UnregisterWaitEx',0,b'\x00\x00\x46\x23WSARecv',0,b'\xFF\xFF\xFF\x1FWT_EXECUTEINWAITTHREAD',4,b'\xFF\xFF\xFF\x1FWT_EXECUTEONLYONCE',8,b'\x00\x00\x90\x23WaitForMultipleObjects',0,b'\x00\x00\x96\x23WaitForSingleObject',0,b'\x00\x00\x77\x23WriteFile',0,b'\x00\x00\x8A\x23_get_osfhandle',0,b'\x00\x00\x17\x23_getch',0,b'\x00\x00\x17\x23_getche',0,b'\x00\x00\xA4\x23_getwch',0,b'\x00\x00\xA4\x23_getwche',0,b'\x00\x00\x17\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\xA6\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\xA1\x23_ungetwch',0,b'\x00\x00\xC8\x23socket',0), - _struct_unions = ((b'\x00\x00\x00\xFE\x00\x00\x00\x03$1',b'\x00\x00\xFD\x11DUMMYSTRUCTNAME',b'\x00\x00\x11\x11Pointer'),(b'\x00\x00\x00\xFD\x00\x00\x00\x02$2',b'\x00\x00\x13\x11Offset',b'\x00\x00\x13\x11OffsetHigh'),(b'\x00\x00\x00\xF5\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x11\x11hProcess',b'\x00\x00\x11\x11hThread',b'\x00\x00\x13\x11dwProcessId',b'\x00\x00\x13\x11dwThreadId'),(b'\x00\x00\x00\xF9\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x13\x11cb',b'\x00\x00\x1A\x11lpReserved',b'\x00\x00\x1A\x11lpDesktop',b'\x00\x00\x1A\x11lpTitle',b'\x00\x00\x13\x11dwX',b'\x00\x00\x13\x11dwY',b'\x00\x00\x13\x11dwXSize',b'\x00\x00\x13\x11dwYSize',b'\x00\x00\x13\x11dwXCountChars',b'\x00\x00\x13\x11dwYCountChars',b'\x00\x00\x13\x11dwFillAttribute',b'\x00\x00\x13\x11dwFlags',b'\x00\x00\xA2\x11wShowWindow',b'\x00\x00\xA2\x11cbReserved2',b'\x00\x00\xFF\x11lpReserved2',b'\x00\x00\x11\x11hStdInput',b'\x00\x00\x11\x11hStdOutput',b'\x00\x00\x11\x11hStdError'),(b'\x00\x00\x00\xF4\x00\x00\x00\x02_OVERLAPPED',b'\x00\x00\x13\x11Internal',b'\x00\x00\x13\x11InternalHigh',b'\x00\x00\xFE\x11DUMMYUNIONNAME',b'\x00\x00\x11\x11hEvent'),(b'\x00\x00\x00\xF7\x00\x00\x00\x02_PostCallbackData',b'\x00\x00\x11\x11hCompletionPort',b'\x00\x00\x15\x11Overlapped'),(b'\x00\x00\x00\xF8\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x13\x11nLength',b'\x00\x00\x11\x11lpSecurityDescriptor',b'\x00\x00\x01\x11bInheritHandle'),(b'\x00\x00\x00\xFA\x00\x00\x00\x02_WSABUF',b'\x00\x00\x13\x11len',b'\x00\x00\x1A\x11buf')), - _typenames = (b'\x00\x00\x00\xFCLPFN_DISCONNECTEX',b'\x00\x00\x00\x15LPOVERLAPPED',b'\x00\x00\x00\x4DLPOVERLAPPED_COMPLETION_ROUTINE',b'\x00\x00\x00\x23LPPROCESS_INFORMATION',b'\x00\x00\x00\xF6LPPostCallbackData',b'\x00\x00\x00\xAALPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x22LPSTARTUPINFO',b'\x00\x00\x00\x48LPWSABUF',b'\x00\x00\x00\xF4OVERLAPPED',b'\x00\x00\x00\xF5PROCESS_INFORMATION',b'\x00\x00\x00\xAAPSECURITY_ATTRIBUTES',b'\x00\x00\x00\xF7PostCallbackData',b'\x00\x00\x00\xF8SECURITY_ATTRIBUTES',b'\x00\x00\x00\x11SOCKET',b'\x00\x00\x00\xF9STARTUPINFO',b'\x00\x00\x00\x2EWAITORTIMERCALLBACK',b'\x00\x00\x00\xFAWSABUF',b'\x00\x00\x00\xA2wint_t'), + _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x0A\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x13\x03\x00\x00\xFD\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x04\x03\x00\x00\x1A\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x1A\x11\x00\x00\x1A\x11\x00\x01\x02\x03\x00\x00\xFE\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x03\x00\x00\x26\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x26\x11\x00\x00\x11\x11\x00\x00\xF9\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x14\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x03\x03\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\xF4\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x48\x11\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x4D\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x14\x03\x00\x00\x15\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x26\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x0A\x03\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\xB0\x03\x00\x00\x88\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x88\x11\x00\x00\x88\x11\x00\x00\x22\x11\x00\x00\x23\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x5A\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x0A\x01\x00\x00\x26\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x13\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x11\x11\x00\x00\x88\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x02\x0F\x00\x00\xAB\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\xAB\x0D\x00\x00\x00\x0F\x00\x00\xAB\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x01\x01\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x01\x04\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xB3\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xB0\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xB6\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xB3\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xB6\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xB3\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x88\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xB3\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xBC\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xB3\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x01\x0A\x0D\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x00\x0F\x00\x01\x0A\x0D\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x04\x09\x00\x00\x02\x09\x00\x01\x00\x03\x00\x00\x05\x09\x00\x00\x06\x09\x00\x00\x03\x09\x00\x00\x07\x09\x00\x00\x02\x01\x00\x00\x40\x03\x00\x00\x01\x09\x00\x00\x00\x09\x00\x01\x09\x03\x00\x00\x04\x01\x00\x00\x00\x01', + _globals = (b'\x00\x00\x33\x23CancelIo',0,b'\x00\x00\x36\x23CancelIoEx',0,b'\x00\x00\x33\x23CloseHandle',0,b'\x00\x00\x36\x23ConnectNamedPipe',0,b'\x00\x00\xB2\x23CreateEventA',0,b'\x00\x00\xB8\x23CreateEventW',0,b'\x00\x00\xBE\x23CreateFileA',0,b'\x00\x00\xEB\x23CreateFileW',0,b'\x00\x00\xD9\x23CreateIoCompletionPort',0,b'\x00\x00\xC7\x23CreateNamedPipeA',0,b'\x00\x00\xE1\x23CreateNamedPipeW',0,b'\x00\x00\x25\x23CreatePipe',0,b'\x00\x00\x19\x23CreateProcessA',0,b'\x00\x00\x87\x23CreateProcessW',0,b'\x00\x00\x77\x23DuplicateHandle',0,b'\x00\x00\xDF\x23GetCurrentProcess',0,b'\x00\x00\x5C\x23GetExitCodeProcess',0,b'\x00\x00\xA8\x23GetLastError',0,b'\x00\x00\xA3\x23GetModuleFileNameW',0,b'\x00\x00\x3A\x23GetOverlappedResult',0,b'\x00\x00\x60\x23GetQueuedCompletionStatus',0,b'\x00\x00\xD6\x23GetStdHandle',0,b'\x00\x00\xA8\x23GetVersion',0,b'\x00\x00\x6D\x23PostQueuedCompletionStatus',0,b'\x00\x00\x10\x23ReadFile',0,b'\x00\x00\x2B\x23RegisterWaitForSingleObject',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\x96\x23SetErrorMode',0,b'\x00\x00\x33\x23SetEvent',0,b'\x00\x00\x67\x23SetNamedPipeHandleState',0,b'\x00\x00\x58\x23TerminateProcess',0,b'\x00\x00\x33\x23UnregisterWait',0,b'\x00\x00\x73\x23UnregisterWaitEx',0,b'\x00\x00\x46\x23WSARecv',0,b'\x00\x00\x4F\x23WSASend',0,b'\xFF\xFF\xFF\x1FWT_EXECUTEINWAITTHREAD',4,b'\xFF\xFF\xFF\x1FWT_EXECUTEONLYONCE',8,b'\x00\x00\x99\x23WaitForMultipleObjects',0,b'\x00\x00\x9F\x23WaitForSingleObject',0,b'\x00\x00\x80\x23WriteFile',0,b'\x00\x00\x93\x23_get_osfhandle',0,b'\x00\x00\x17\x23_getch',0,b'\x00\x00\x17\x23_getche',0,b'\x00\x00\xAD\x23_getwch',0,b'\x00\x00\xAD\x23_getwche',0,b'\x00\x00\x17\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\xAF\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\xAA\x23_ungetwch',0,b'\x00\x00\xD1\x23socket',0), + _struct_unions = ((b'\x00\x00\x01\x07\x00\x00\x00\x03$1',b'\x00\x01\x06\x11DUMMYSTRUCTNAME',b'\x00\x00\x11\x11Pointer'),(b'\x00\x00\x01\x06\x00\x00\x00\x02$2',b'\x00\x00\x13\x11Offset',b'\x00\x00\x13\x11OffsetHigh'),(b'\x00\x00\x00\xFE\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x11\x11hProcess',b'\x00\x00\x11\x11hThread',b'\x00\x00\x13\x11dwProcessId',b'\x00\x00\x13\x11dwThreadId'),(b'\x00\x00\x01\x02\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x13\x11cb',b'\x00\x00\x1A\x11lpReserved',b'\x00\x00\x1A\x11lpDesktop',b'\x00\x00\x1A\x11lpTitle',b'\x00\x00\x13\x11dwX',b'\x00\x00\x13\x11dwY',b'\x00\x00\x13\x11dwXSize',b'\x00\x00\x13\x11dwYSize',b'\x00\x00\x13\x11dwXCountChars',b'\x00\x00\x13\x11dwYCountChars',b'\x00\x00\x13\x11dwFillAttribute',b'\x00\x00\x13\x11dwFlags',b'\x00\x00\xAB\x11wShowWindow',b'\x00\x00\xAB\x11cbReserved2',b'\x00\x01\x08\x11lpReserved2',b'\x00\x00\x11\x11hStdInput',b'\x00\x00\x11\x11hStdOutput',b'\x00\x00\x11\x11hStdError'),(b'\x00\x00\x00\xFD\x00\x00\x00\x02_OVERLAPPED',b'\x00\x00\x13\x11Internal',b'\x00\x00\x13\x11InternalHigh',b'\x00\x01\x07\x11DUMMYUNIONNAME',b'\x00\x00\x11\x11hEvent'),(b'\x00\x00\x01\x00\x00\x00\x00\x02_PostCallbackData',b'\x00\x00\x11\x11hCompletionPort',b'\x00\x00\x15\x11Overlapped'),(b'\x00\x00\x01\x01\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x13\x11nLength',b'\x00\x00\x11\x11lpSecurityDescriptor',b'\x00\x00\x01\x11bInheritHandle'),(b'\x00\x00\x01\x03\x00\x00\x00\x02_WSABUF',b'\x00\x00\x13\x11len',b'\x00\x00\x1A\x11buf')), + _typenames = (b'\x00\x00\x01\x05LPFN_DISCONNECTEX',b'\x00\x00\x00\x15LPOVERLAPPED',b'\x00\x00\x00\x4DLPOVERLAPPED_COMPLETION_ROUTINE',b'\x00\x00\x00\x23LPPROCESS_INFORMATION',b'\x00\x00\x00\xFFLPPostCallbackData',b'\x00\x00\x00\xB3LPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x22LPSTARTUPINFO',b'\x00\x00\x00\x48LPWSABUF',b'\x00\x00\x00\xFDOVERLAPPED',b'\x00\x00\x00\xFEPROCESS_INFORMATION',b'\x00\x00\x00\xB3PSECURITY_ATTRIBUTES',b'\x00\x00\x01\x00PostCallbackData',b'\x00\x00\x01\x01SECURITY_ATTRIBUTES',b'\x00\x00\x00\x11SOCKET',b'\x00\x00\x01\x02STARTUPINFO',b'\x00\x00\x00\x2EWAITORTIMERCALLBACK',b'\x00\x00\x01\x03WSABUF',b'\x00\x00\x00\xABwint_t'), ) diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py --- a/lib_pypy/_winapi.py +++ b/lib_pypy/_winapi.py @@ -346,4 +346,5 @@ OPEN_ALWAYS = 4 TRUNCATE_EXISTING = 5 -MAXIMUM_WAIT_OBJECTS = 64 \ No newline at end of file +MAXIMUM_WAIT_OBJECTS = 64 + From pypy.commits at gmail.com Sun Mar 17 05:29:09 2019 From: pypy.commits at gmail.com (andrewjlawrence) Date: Sun, 17 Mar 2019 02:29:09 -0700 (PDT) Subject: [pypy-commit] pypy winoverlapped: Work in progress implementing AcceptEx Message-ID: <5c8e1365.1c69fb81.d4382.99b9@mx.google.com> Author: andrewjlawrence Branch: winoverlapped Changeset: r96331:8da7c8e10b0a Date: 2019-03-17 09:28 +0000 http://bitbucket.org/pypy/pypy/changeset/8da7c8e10b0a/ Log: Work in progress implementing AcceptEx diff --git a/lib_pypy/_overlapped.py b/lib_pypy/_overlapped.py --- a/lib_pypy/_overlapped.py +++ b/lib_pypy/_overlapped.py @@ -15,6 +15,8 @@ _winsock2 = _ffi.dlopen('Ws2_32') +_mswsock = _ffi.dlopen('Mswsock') + GetVersion = _kernel32.GetVersion NULL = _ffi.NULL @@ -30,9 +32,29 @@ SOCKET_ERROR = -1 -# +AF_INET = 2 +SOCK_STREAM = 1 +IPPROTO_TCP = 6 + +INVALID_SOCKET = -1 + +IOC_OUT = 0x40000000 +IOC_IN = 0x80000000 +IOC_INOUT = IOC_IN | IOC_OUT +IOC_WS2 = 0x08000000 + +def _WSAIORW(x, y): + return IOC_INOUT | x | y + +WSAID_ACCEPTEX = _ffi.new("GUID[1]") +WSAID_ACCEPTEX[0].Data1 = 0xb5367df1 +WSAID_ACCEPTEX[0].Data2 = 0xcbac +WSAID_ACCEPTEX[0].Data3 = 0x11cf +WSAID_ACCEPTEX[0].Data4 = [0x95,0xca,0x00,0x80,0x5f,0x48,0xa1,0x92] + +SIO_GET_EXTENSION_FUNCTION_POINTER = _WSAIORW(IOC_WS2,6) + # Status Codes -# STATUS_PENDING = 0x00000103 DisconnectEx = _ffi.NULL @@ -64,6 +86,15 @@ TYPE_WAIT_NAMED_PIPE_AND_CONNECT = 9 TYPE_TRANSMIT_FILE = 10 +_accept_ex = _ffi.new("AcceptExPtr[1]") + +def initiailize_function_ptrs(): + s = _winsock2.socket(AF_INET, SOCK_STREAM, IPPROTO_TCP) + if s == INVALID_SOCKET: + raise _winapi._WinError() + + + class Overlapped(object): def __init__(self, event=_ffi.NULL): self.overlapped = _ffi.new('OVERLAPPED[1]') @@ -198,7 +229,6 @@ wsabuff[0].buf = _ffi.new("CHAR[]", bufobj) nwritten = _ffi.new("LPDWORD") - result = _winsock2.WSASend(handle, wsabuff, _int2dword(1), nwritten, flags, self.overlapped, _ffi.NULL) if result == SOCKET_ERROR: @@ -295,6 +325,7 @@ def address(self): return self.overlapped + AcceptEx = None def SetEvent(handle): ret = _kernel32.SetEvent(handle) diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py --- a/lib_pypy/_pypy_winbase_build.py +++ b/lib_pypy/_pypy_winbase_build.py @@ -165,10 +165,20 @@ typedef HANDLE SOCKET; SOCKET __stdcall socket(int, int, int); typedef BOOL (__stdcall * LPFN_DISCONNECTEX) (SOCKET, LPOVERLAPPED, DWORD, DWORD); -typedef VOID (*LPOVERLAPPED_COMPLETION_ROUTINE) (DWORD, DWORD, LPVOID); - +typedef VOID (*LPOVERLAPPED_COMPLETION_ROUTINE) (DWORD, DWORD, LPVOID); +typedef BOOL (WINAPI* AcceptExPtr)(SOCKET, SOCKET, PVOID, DWORD, DWORD, DWORD, LPDWORD, LPOVERLAPPED); +typedef BOOL (WINAPI AcceptEx)(SOCKET, SOCKET, PVOID, DWORD, DWORD, DWORD, LPDWORD, LPOVERLAPPED); int __stdcall WSARecv(SOCKET, LPWSABUF, DWORD, LPDWORD, LPDWORD, LPOVERLAPPED, LPOVERLAPPED_COMPLETION_ROUTINE); int __stdcall WSASend(SOCKET, LPWSABUF, DWORD, LPDWORD, DWORD, LPOVERLAPPED, LPOVERLAPPED_COMPLETION_ROUTINE); +int __stdcall WSAIoctl(SOCKET, DWORD, LPVOID, DWORD, LPVOID, DWORD, LPDWORD, LPOVERLAPPED, LPOVERLAPPED_COMPLETION_ROUTINE); + + +typedef struct _GUID { + DWORD Data1; + WORD Data2; + WORD Data3; + BYTE Data4[8]; +} GUID; """) diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py --- a/lib_pypy/_pypy_winbase_cffi.py +++ b/lib_pypy/_pypy_winbase_cffi.py @@ -3,8 +3,8 @@ ffi = _cffi_backend.FFI('_pypy_winbase_cffi', _version = 0x2601, - _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x0A\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x13\x03\x00\x00\xFD\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x04\x03\x00\x00\x1A\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x1A\x11\x00\x00\x1A\x11\x00\x01\x02\x03\x00\x00\xFE\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x03\x00\x00\x26\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x26\x11\x00\x00\x11\x11\x00\x00\xF9\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x14\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x03\x03\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\xF4\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x48\x11\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x4D\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x14\x03\x00\x00\x15\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x26\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x0A\x03\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\xB0\x03\x00\x00\x88\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x88\x11\x00\x00\x88\x11\x00\x00\x22\x11\x00\x00\x23\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x5A\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x0A\x01\x00\x00\x26\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x13\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x11\x11\x00\x00\x88\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x02\x0F\x00\x00\xAB\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\xAB\x0D\x00\x00\x00\x0F\x00\x00\xAB\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x01\x01\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x01\x04\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xB3\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xB0\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xB6\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xB3\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xB6\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xB3\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x88\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xB3\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xBC\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xB3\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x01\x0A\x0D\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x00\x0F\x00\x01\x0A\x0D\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x04\x09\x00\x00\x02\x09\x00\x01\x00\x03\x00\x00\x05\x09\x00\x00\x06\x09\x00\x00\x03\x09\x00\x00\x07\x09\x00\x00\x02\x01\x00\x00\x40\x03\x00\x00\x01\x09\x00\x00\x00\x09\x00\x01\x09\x03\x00\x00\x04\x01\x00\x00\x00\x01', - _globals = (b'\x00\x00\x33\x23CancelIo',0,b'\x00\x00\x36\x23CancelIoEx',0,b'\x00\x00\x33\x23CloseHandle',0,b'\x00\x00\x36\x23ConnectNamedPipe',0,b'\x00\x00\xB2\x23CreateEventA',0,b'\x00\x00\xB8\x23CreateEventW',0,b'\x00\x00\xBE\x23CreateFileA',0,b'\x00\x00\xEB\x23CreateFileW',0,b'\x00\x00\xD9\x23CreateIoCompletionPort',0,b'\x00\x00\xC7\x23CreateNamedPipeA',0,b'\x00\x00\xE1\x23CreateNamedPipeW',0,b'\x00\x00\x25\x23CreatePipe',0,b'\x00\x00\x19\x23CreateProcessA',0,b'\x00\x00\x87\x23CreateProcessW',0,b'\x00\x00\x77\x23DuplicateHandle',0,b'\x00\x00\xDF\x23GetCurrentProcess',0,b'\x00\x00\x5C\x23GetExitCodeProcess',0,b'\x00\x00\xA8\x23GetLastError',0,b'\x00\x00\xA3\x23GetModuleFileNameW',0,b'\x00\x00\x3A\x23GetOverlappedResult',0,b'\x00\x00\x60\x23GetQueuedCompletionStatus',0,b'\x00\x00\xD6\x23GetStdHandle',0,b'\x00\x00\xA8\x23GetVersion',0,b'\x00\x00\x6D\x23PostQueuedCompletionStatus',0,b'\x00\x00\x10\x23ReadFile',0,b'\x00\x00\x2B\x23RegisterWaitForSingleObject',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\x96\x23SetErrorMode',0,b'\x00\x00\x33\x23SetEvent',0,b'\x00\x00\x67\x23SetNamedPipeHandleState',0,b'\x00\x00\x58\x23TerminateProcess',0,b'\x00\x00\x33\x23UnregisterWait',0,b'\x00\x00\x73\x23UnregisterWaitEx',0,b'\x00\x00\x46\x23WSARecv',0,b'\x00\x00\x4F\x23WSASend',0,b'\xFF\xFF\xFF\x1FWT_EXECUTEINWAITTHREAD',4,b'\xFF\xFF\xFF\x1FWT_EXECUTEONLYONCE',8,b'\x00\x00\x99\x23WaitForMultipleObjects',0,b'\x00\x00\x9F\x23WaitForSingleObject',0,b'\x00\x00\x80\x23WriteFile',0,b'\x00\x00\x93\x23_get_osfhandle',0,b'\x00\x00\x17\x23_getch',0,b'\x00\x00\x17\x23_getche',0,b'\x00\x00\xAD\x23_getwch',0,b'\x00\x00\xAD\x23_getwche',0,b'\x00\x00\x17\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\xAF\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\xAA\x23_ungetwch',0,b'\x00\x00\xD1\x23socket',0), - _struct_unions = ((b'\x00\x00\x01\x07\x00\x00\x00\x03$1',b'\x00\x01\x06\x11DUMMYSTRUCTNAME',b'\x00\x00\x11\x11Pointer'),(b'\x00\x00\x01\x06\x00\x00\x00\x02$2',b'\x00\x00\x13\x11Offset',b'\x00\x00\x13\x11OffsetHigh'),(b'\x00\x00\x00\xFE\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x11\x11hProcess',b'\x00\x00\x11\x11hThread',b'\x00\x00\x13\x11dwProcessId',b'\x00\x00\x13\x11dwThreadId'),(b'\x00\x00\x01\x02\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x13\x11cb',b'\x00\x00\x1A\x11lpReserved',b'\x00\x00\x1A\x11lpDesktop',b'\x00\x00\x1A\x11lpTitle',b'\x00\x00\x13\x11dwX',b'\x00\x00\x13\x11dwY',b'\x00\x00\x13\x11dwXSize',b'\x00\x00\x13\x11dwYSize',b'\x00\x00\x13\x11dwXCountChars',b'\x00\x00\x13\x11dwYCountChars',b'\x00\x00\x13\x11dwFillAttribute',b'\x00\x00\x13\x11dwFlags',b'\x00\x00\xAB\x11wShowWindow',b'\x00\x00\xAB\x11cbReserved2',b'\x00\x01\x08\x11lpReserved2',b'\x00\x00\x11\x11hStdInput',b'\x00\x00\x11\x11hStdOutput',b'\x00\x00\x11\x11hStdError'),(b'\x00\x00\x00\xFD\x00\x00\x00\x02_OVERLAPPED',b'\x00\x00\x13\x11Internal',b'\x00\x00\x13\x11InternalHigh',b'\x00\x01\x07\x11DUMMYUNIONNAME',b'\x00\x00\x11\x11hEvent'),(b'\x00\x00\x01\x00\x00\x00\x00\x02_PostCallbackData',b'\x00\x00\x11\x11hCompletionPort',b'\x00\x00\x15\x11Overlapped'),(b'\x00\x00\x01\x01\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x13\x11nLength',b'\x00\x00\x11\x11lpSecurityDescriptor',b'\x00\x00\x01\x11bInheritHandle'),(b'\x00\x00\x01\x03\x00\x00\x00\x02_WSABUF',b'\x00\x00\x13\x11len',b'\x00\x00\x1A\x11buf')), - _typenames = (b'\x00\x00\x01\x05LPFN_DISCONNECTEX',b'\x00\x00\x00\x15LPOVERLAPPED',b'\x00\x00\x00\x4DLPOVERLAPPED_COMPLETION_ROUTINE',b'\x00\x00\x00\x23LPPROCESS_INFORMATION',b'\x00\x00\x00\xFFLPPostCallbackData',b'\x00\x00\x00\xB3LPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x22LPSTARTUPINFO',b'\x00\x00\x00\x48LPWSABUF',b'\x00\x00\x00\xFDOVERLAPPED',b'\x00\x00\x00\xFEPROCESS_INFORMATION',b'\x00\x00\x00\xB3PSECURITY_ATTRIBUTES',b'\x00\x00\x01\x00PostCallbackData',b'\x00\x00\x01\x01SECURITY_ATTRIBUTES',b'\x00\x00\x00\x11SOCKET',b'\x00\x00\x01\x02STARTUPINFO',b'\x00\x00\x00\x2EWAITORTIMERCALLBACK',b'\x00\x00\x01\x03WSABUF',b'\x00\x00\x00\xABwint_t'), + _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x23\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x13\x03\x00\x01\x13\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x1A\x03\x00\x00\x1A\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x1A\x11\x00\x00\x1A\x11\x00\x01\x18\x03\x00\x01\x14\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x03\x00\x00\x26\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x26\x11\x00\x00\x11\x11\x00\x01\x0E\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x14\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x19\x03\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x15\x11\x00\x01\x09\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x48\x11\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x4D\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x14\x03\x00\x00\x15\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\x4D\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x26\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x23\x03\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\xC5\x03\x00\x00\x9D\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x9D\x11\x00\x00\x9D\x11\x00\x00\x22\x11\x00\x00\x23\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x5A\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x0A\x01\x00\x00\x26\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x13\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x11\x11\x00\x00\x9D\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x02\x0F\x00\x00\xC0\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\xC0\x0D\x00\x00\x00\x0F\x00\x00\xC0\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x01\x17\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x01\x1A\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xC8\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xC5\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xCB\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xC8\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xCB\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xC8\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x9D\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xC8\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xD1\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xC8\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x01\x23\x0D\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x00\x0F\x00\x01\x23\x0D\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x04\x09\x00\x00\x05\x09\x00\x00\x02\x09\x00\x01\x16\x03\x00\x00\x06\x09\x00\x00\x07\x09\x00\x00\x03\x09\x00\x00\x08\x09\x00\x00\x02\x01\x00\x00\x40\x03\x00\x00\x82\x03\x00\x00\x01\x09\x00\x00\x00\x09\x00\x01\x20\x03\x00\x00\x04\x01\x00\x01\x20\x05\x00\x00\x00\x08\x00\x00\x00\x01', + _globals = (b'\x00\x00\x33\x23CancelIo',0,b'\x00\x00\x36\x23CancelIoEx',0,b'\x00\x00\x33\x23CloseHandle',0,b'\x00\x00\x36\x23ConnectNamedPipe',0,b'\x00\x00\xC7\x23CreateEventA',0,b'\x00\x00\xCD\x23CreateEventW',0,b'\x00\x00\xD3\x23CreateFileA',0,b'\x00\x01\x00\x23CreateFileW',0,b'\x00\x00\xEE\x23CreateIoCompletionPort',0,b'\x00\x00\xDC\x23CreateNamedPipeA',0,b'\x00\x00\xF6\x23CreateNamedPipeW',0,b'\x00\x00\x25\x23CreatePipe',0,b'\x00\x00\x19\x23CreateProcessA',0,b'\x00\x00\x9C\x23CreateProcessW',0,b'\x00\x00\x8C\x23DuplicateHandle',0,b'\x00\x00\xF4\x23GetCurrentProcess',0,b'\x00\x00\x5C\x23GetExitCodeProcess',0,b'\x00\x00\xBD\x23GetLastError',0,b'\x00\x00\xB8\x23GetModuleFileNameW',0,b'\x00\x00\x3A\x23GetOverlappedResult',0,b'\x00\x00\x60\x23GetQueuedCompletionStatus',0,b'\x00\x00\xEB\x23GetStdHandle',0,b'\x00\x00\xBD\x23GetVersion',0,b'\x00\x00\x6D\x23PostQueuedCompletionStatus',0,b'\x00\x00\x10\x23ReadFile',0,b'\x00\x00\x2B\x23RegisterWaitForSingleObject',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\xAB\x23SetErrorMode',0,b'\x00\x00\x33\x23SetEvent',0,b'\x00\x00\x67\x23SetNamedPipeHandleState',0,b'\x00\x00\x58\x23TerminateProcess',0,b'\x00\x00\x33\x23UnregisterWait',0,b'\x00\x00\x7E\x23UnregisterWaitEx',0,b'\x00\x00\x73\x23WSAIoctl',0,b'\x00\x00\x46\x23WSARecv',0,b'\x00\x00\x4F\x23WSASend',0,b'\xFF\xFF\xFF\x1FWT_EXECUTEINWAITTHREAD',4,b'\xFF\xFF\xFF\x1FWT_EXECUTEONLYONCE',8,b'\x00\x00\xAE\x23WaitForMultipleObjects',0,b'\x00\x00\xB4\x23WaitForSingleObject',0,b'\x00\x00\x95\x23WriteFile',0,b'\x00\x00\xA8\x23_get_osfhandle',0,b'\x00\x00\x17\x23_getch',0,b'\x00\x00\x17\x23_getche',0,b'\x00\x00\xC2\x23_getwch',0,b'\x00\x00\xC2\x23_getwche',0,b'\x00\x00\x17\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\xC4\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\xBF\x23_ungetwch',0,b'\x00\x00\xE6\x23socket',0), + _struct_unions = ((b'\x00\x00\x01\x1E\x00\x00\x00\x03$1',b'\x00\x01\x1D\x11DUMMYSTRUCTNAME',b'\x00\x00\x11\x11Pointer'),(b'\x00\x00\x01\x1D\x00\x00\x00\x02$2',b'\x00\x00\x13\x11Offset',b'\x00\x00\x13\x11OffsetHigh'),(b'\x00\x00\x01\x14\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x11\x11hProcess',b'\x00\x00\x11\x11hThread',b'\x00\x00\x13\x11dwProcessId',b'\x00\x00\x13\x11dwThreadId'),(b'\x00\x00\x01\x18\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x13\x11cb',b'\x00\x00\x1A\x11lpReserved',b'\x00\x00\x1A\x11lpDesktop',b'\x00\x00\x1A\x11lpTitle',b'\x00\x00\x13\x11dwX',b'\x00\x00\x13\x11dwY',b'\x00\x00\x13\x11dwXSize',b'\x00\x00\x13\x11dwYSize',b'\x00\x00\x13\x11dwXCountChars',b'\x00\x00\x13\x11dwYCountChars',b'\x00\x00\x13\x11dwFillAttribute',b'\x00\x00\x13\x11dwFlags',b'\x00\x00\xC0\x11wShowWindow',b'\x00\x00\xC0\x11cbReserved2',b'\x00\x01\x1F\x11lpReserved2',b'\x00\x00\x11\x11hStdInput',b'\x00\x00\x11\x11hStdOutput',b'\x00\x00\x11\x11hStdError'),(b'\x00\x00\x01\x12\x00\x00\x00\x02_GUID',b'\x00\x00\x13\x11Data1',b'\x00\x00\xC0\x11Data2',b'\x00\x00\xC0\x11Data3',b'\x00\x01\x21\x11Data4'),(b'\x00\x00\x01\x13\x00\x00\x00\x02_OVERLAPPED',b'\x00\x00\x13\x11Internal',b'\x00\x00\x13\x11InternalHigh',b'\x00\x01\x1E\x11DUMMYUNIONNAME',b'\x00\x00\x11\x11hEvent'),(b'\x00\x00\x01\x16\x00\x00\x00\x02_PostCallbackData',b'\x00\x00\x11\x11hCompletionPort',b'\x00\x00\x15\x11Overlapped'),(b'\x00\x00\x01\x17\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x13\x11nLength',b'\x00\x00\x11\x11lpSecurityDescriptor',b'\x00\x00\x01\x11bInheritHandle'),(b'\x00\x00\x01\x19\x00\x00\x00\x02_WSABUF',b'\x00\x00\x13\x11len',b'\x00\x00\x1A\x11buf')), + _typenames = (b'\x00\x00\x00\x82AcceptEx',b'\x00\x00\x01\x1CAcceptExPtr',b'\x00\x00\x01\x12GUID',b'\x00\x00\x01\x1BLPFN_DISCONNECTEX',b'\x00\x00\x00\x15LPOVERLAPPED',b'\x00\x00\x00\x4DLPOVERLAPPED_COMPLETION_ROUTINE',b'\x00\x00\x00\x23LPPROCESS_INFORMATION',b'\x00\x00\x01\x15LPPostCallbackData',b'\x00\x00\x00\xC8LPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x22LPSTARTUPINFO',b'\x00\x00\x00\x48LPWSABUF',b'\x00\x00\x01\x13OVERLAPPED',b'\x00\x00\x01\x14PROCESS_INFORMATION',b'\x00\x00\x00\xC8PSECURITY_ATTRIBUTES',b'\x00\x00\x01\x16PostCallbackData',b'\x00\x00\x01\x17SECURITY_ATTRIBUTES',b'\x00\x00\x00\x11SOCKET',b'\x00\x00\x01\x18STARTUPINFO',b'\x00\x00\x00\x2EWAITORTIMERCALLBACK',b'\x00\x00\x01\x19WSABUF',b'\x00\x00\x00\xC0wint_t'), ) From pypy.commits at gmail.com Sun Mar 17 06:37:42 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 17 Mar 2019 03:37:42 -0700 (PDT) Subject: [pypy-commit] buildbot default: add benchmarker_32, benchmarker; remove 32 bit JITBENCH Message-ID: <5c8e2376.1c69fb81.ec8df.4e11@mx.google.com> Author: Matti Picus Branch: Changeset: r1076:1c75acbd3686 Date: 2019-03-17 12:37 +0200 http://bitbucket.org/pypy/buildbot/changeset/1c75acbd3686/ Log: add benchmarker_32, benchmarker; remove 32 bit JITBENCH diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -217,7 +217,6 @@ JITWIN32 = "pypy-c-jit-win-x86-32" JITONLYLINUXPPC64 = "jitonly-own-linux-ppc-64" -JITBENCH = "jit-benchmark-linux-x86-32" JITBENCH64 = "jit-benchmark-linux-x86-64" CPYTHON_64 = "cpython-2-benchmark-x86-64" NUMPY_64 = "numpy-compatibility-linux-x86-64" @@ -286,32 +285,32 @@ ], 'schedulers': [ - # the benchmarks run on tannit and (planned) speed-old.python.org. + # the benchmarks run on benchmarker and (planned) speed-old.python.org. # All the other linux tests run on bencher4.soft-dev.org. Nightly("nightly-0-00", [ # linux tests - LINUX32OWN, # on bencher4_32, uses all cores + LINUX32OWN, # on benchmarker4_32, uses all cores LINUX64OWN, # on bencher4, uses all cores WIN32OWN, # on SalsaSalsa - JITLINUX32, # on bencher4_32, uses 1 core + JITLINUX32, # on benchmarker4_32, uses 1 core JITLINUX64, # on bencher4, uses 1 core - #APPLVLLINUX32, # on bencher4_32, uses 1 core + #APPLVLLINUX32, #APPLVLLINUX64, # on bencher4, uses 1 core # other platforms #MACOSX32, # on minime JITWIN32, # on SalsaSalsa - #JITFREEBSD764, # on headless - #JITFREEBSD864, # on ananke - #JITFREEBSD964, # on tavendo + #JITFREEBSD764, + #JITFREEBSD864, + #JITFREEBSD964, JITMACOSX64, # on xerxes # buildbot selftest - #PYPYBUILDBOT # on cobra + #PYPYBUILDBOT, ], branch='default', hour=0, minute=0, onlyIfChanged=True, ), Nightly("nightly-0-01", [ - LINUX32RPYTHON, # on bencher4_32, uses all cores + LINUX32RPYTHON, # on benchermarker_32, uses all cores LINUX64RPYTHON, # on bencher4, uses all cores WIN32RPYTHON, # on SalsaSalsa LINUX_S390XRPYTHON, @@ -321,7 +320,6 @@ ), Nightly("nightly-1-00", [ - #JITBENCH, # on tannit32, uses 1 core (in part exclusively) #JITBENCH64, # on tannit64, uses 1 core (in part exclusively) #JITBENCH64_NEW, # on speed64, uses 1 core (in part exclusively) @@ -392,7 +390,6 @@ #JITFREEBSD964, JITONLYLINUXPPC64, - JITBENCH, JITBENCH64, JITBENCH64_NEW, NUMPY_64, @@ -418,14 +415,14 @@ 'builders': [ {"name": LINUX32OWN, - "slavenames": ["bencher4_32", "salsa_32"], + "slavenames": ["salsa_32", "benchmarker_32"], "builddir": LINUX32OWN, "factory": pypyOwnTestFactory, "category": 'linux32', "locks": [TannitCPU.access('counting')], }, {"name": LINUX32RPYTHON, - "slavenames": ["bencher4_32", "salsa_32"], + "slavenames": ["salsa_32", "benchmarker_32"], "builddir": LINUX32RPYTHON, "factory": pypyRPythonTestFactory, "category": 'linux32', @@ -449,7 +446,7 @@ }, {"name": APPLVLLINUX32, #"slavenames": ["allegro32"], - "slavenames": ["bencher4_32"], + "slavenames": ["benchmarker_32"], "builddir": APPLVLLINUX32, "factory": pypyTranslatedAppLevelTestFactory, 'category': 'linux32', @@ -481,7 +478,7 @@ }, {"name" : JITLINUX32, #"slavenames": ["allegro32"], - "slavenames": ["bencher4_32", "salsa_32"], + "slavenames": ["bencher4_32", "salsa_32", "benchmarker_32"], 'builddir' : JITLINUX32, 'factory' : pypyJITTranslatedTestFactory, 'category' : 'linux32', @@ -495,15 +492,8 @@ 'category': 'linux64', "locks": [Bencher4Lock.access('counting')], }, - {"name": JITBENCH, - "slavenames": ["tannit32"], - "builddir": JITBENCH, - "factory": pypyJITBenchmarkFactory_tannit, - "category": 'benchmark-run', - # the locks are acquired with fine grain inside the build - }, {"name": JITBENCH64, - "slavenames": ["tannit64"], + "slavenames": ["tannit64", "benchmarker"], "builddir": JITBENCH64, "factory": pypyJITBenchmarkFactory64_tannit, "category": "benchmark-run", From pypy.commits at gmail.com Sun Mar 17 07:00:31 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 17 Mar 2019 04:00:31 -0700 (PDT) Subject: [pypy-commit] buildbot default: rename for consistency Message-ID: <5c8e28cf.1c69fb81.6a1bf.6c46@mx.google.com> Author: Matti Picus Branch: Changeset: r1077:c18cf92e03d2 Date: 2019-03-17 13:00 +0200 http://bitbucket.org/pypy/buildbot/changeset/c18cf92e03d2/ Log: rename for consistency diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -415,14 +415,14 @@ 'builders': [ {"name": LINUX32OWN, - "slavenames": ["salsa_32", "benchmarker_32"], + "slavenames": ["salsa_32", "benchmarker32"], "builddir": LINUX32OWN, "factory": pypyOwnTestFactory, "category": 'linux32', "locks": [TannitCPU.access('counting')], }, {"name": LINUX32RPYTHON, - "slavenames": ["salsa_32", "benchmarker_32"], + "slavenames": ["salsa_32", "benchmarker32"], "builddir": LINUX32RPYTHON, "factory": pypyRPythonTestFactory, "category": 'linux32', @@ -446,7 +446,7 @@ }, {"name": APPLVLLINUX32, #"slavenames": ["allegro32"], - "slavenames": ["benchmarker_32"], + "slavenames": ["benchmarker32"], "builddir": APPLVLLINUX32, "factory": pypyTranslatedAppLevelTestFactory, 'category': 'linux32', @@ -478,7 +478,7 @@ }, {"name" : JITLINUX32, #"slavenames": ["allegro32"], - "slavenames": ["bencher4_32", "salsa_32", "benchmarker_32"], + "slavenames": ["bencher4_32", "salsa_32", "benchmarker32"], 'builddir' : JITLINUX32, 'factory' : pypyJITTranslatedTestFactory, 'category' : 'linux32', From pypy.commits at gmail.com Sun Mar 17 09:35:31 2019 From: pypy.commits at gmail.com (andrewjlawrence) Date: Sun, 17 Mar 2019 06:35:31 -0700 (PDT) Subject: [pypy-commit] pypy winoverlapped: Added initialization of dynamically loaded functions. Message-ID: <5c8e4d23.1c69fb81.3f56b.cab1@mx.google.com> Author: andrewjlawrence Branch: winoverlapped Changeset: r96332:ce42458cb91d Date: 2019-03-17 13:33 +0000 http://bitbucket.org/pypy/pypy/changeset/ce42458cb91d/ Log: Added initialization of dynamically loaded functions. diff --git a/lib_pypy/_overlapped.py b/lib_pypy/_overlapped.py --- a/lib_pypy/_overlapped.py +++ b/lib_pypy/_overlapped.py @@ -52,12 +52,26 @@ WSAID_ACCEPTEX[0].Data3 = 0x11cf WSAID_ACCEPTEX[0].Data4 = [0x95,0xca,0x00,0x80,0x5f,0x48,0xa1,0x92] + +WSAID_CONNECTEX = _ffi.new("GUID[1]") +WSAID_CONNECTEX[0].Data1 = 0x25a207b9 +WSAID_CONNECTEX[0].Data2 = 0xddf3 +WSAID_CONNECTEX[0].Data3 = 0x4660 +WSAID_CONNECTEX[0].Data4 = [0x8e,0xe9,0x76,0xe5,0x8c,0x74,0x06,0x3e] + +WSAID_DISCONNECTEX = _ffi.new("GUID[1]") +WSAID_DISCONNECTEX[0].Data1 = 0x7fda2e11 +WSAID_DISCONNECTEX[0].Data2 = 0x8630 +WSAID_DISCONNECTEX[0].Data3 = 0x436f +WSAID_DISCONNECTEX[0].Data4 = [0xa0,0x31,0xf5,0x36,0xa6,0xee,0xc1,0x57] + SIO_GET_EXTENSION_FUNCTION_POINTER = _WSAIORW(IOC_WS2,6) +SO_UPDATE_ACCEPT_CONTEXT = 0x700B + # Status Codes STATUS_PENDING = 0x00000103 -DisconnectEx = _ffi.NULL def _int2intptr(int2cast): return _ffi.cast("ULONG_PTR", int2cast) @@ -86,13 +100,37 @@ TYPE_WAIT_NAMED_PIPE_AND_CONNECT = 9 TYPE_TRANSMIT_FILE = 10 -_accept_ex = _ffi.new("AcceptExPtr[1]") +_accept_ex = _ffi.new("AcceptExPtr*") +_connect_ex = _ffi.new("ConnectExPtr*") +_disconnect_ex = _ffi.new("DisconnectExPtr*") + def initiailize_function_ptrs(): s = _winsock2.socket(AF_INET, SOCK_STREAM, IPPROTO_TCP) + dwBytes = _ffi.new("DWORD[1]", [0]) if s == INVALID_SOCKET: raise _winapi._WinError() + result = _winsock2.WSAIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, \ + WSAID_ACCEPTEX, _ffi.sizeof(WSAID_ACCEPTEX[0]), _accept_ex, \ + _ffi.sizeof(_accept_ex[0]), dwBytes, _ffi.NULL, _ffi.NULL) + if result == INVALID_SOCKET: + raise _winapi._WinError() + + result = _winsock2.WSAIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, \ + WSAID_CONNECTEX, _ffi.sizeof(WSAID_CONNECTEX[0]), _connect_ex, \ + _ffi.sizeof(_connect_ex[0]), dwBytes, _ffi.NULL, _ffi.NULL) + if result == INVALID_SOCKET: + raise _winapi._WinError() + + result = _winsock2.WSAIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, \ + WSAID_DISCONNECTEX, _ffi.sizeof(WSAID_DISCONNECTEX[0]), _disconnect_ex, \ + _ffi.sizeof(_disconnect_ex[0]), dwBytes, _ffi.NULL, _ffi.NULL) + if result == INVALID_SOCKET: + raise _winapi._WinError() + + +initiailize_function_ptrs() class Overlapped(object): @@ -316,6 +354,10 @@ self.type = OverlappedType.TYPE_NOT_STARTED raise _winapi.WinError() + def AcceptEx(self, listensocket, acceptsocket): + xxx + return None + @property def pending(self): return (not HasOverlappedIoCompleted(self.overlapped[0]) and @@ -325,8 +367,6 @@ def address(self): return self.overlapped - AcceptEx = None - def SetEvent(handle): ret = _kernel32.SetEvent(handle) if not ret: diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py --- a/lib_pypy/_pypy_winbase_build.py +++ b/lib_pypy/_pypy_winbase_build.py @@ -166,12 +166,14 @@ SOCKET __stdcall socket(int, int, int); typedef BOOL (__stdcall * LPFN_DISCONNECTEX) (SOCKET, LPOVERLAPPED, DWORD, DWORD); typedef VOID (*LPOVERLAPPED_COMPLETION_ROUTINE) (DWORD, DWORD, LPVOID); -typedef BOOL (WINAPI* AcceptExPtr)(SOCKET, SOCKET, PVOID, DWORD, DWORD, DWORD, LPDWORD, LPOVERLAPPED); -typedef BOOL (WINAPI AcceptEx)(SOCKET, SOCKET, PVOID, DWORD, DWORD, DWORD, LPDWORD, LPOVERLAPPED); + int __stdcall WSARecv(SOCKET, LPWSABUF, DWORD, LPDWORD, LPDWORD, LPOVERLAPPED, LPOVERLAPPED_COMPLETION_ROUTINE); int __stdcall WSASend(SOCKET, LPWSABUF, DWORD, LPDWORD, DWORD, LPOVERLAPPED, LPOVERLAPPED_COMPLETION_ROUTINE); int __stdcall WSAIoctl(SOCKET, DWORD, LPVOID, DWORD, LPVOID, DWORD, LPDWORD, LPOVERLAPPED, LPOVERLAPPED_COMPLETION_ROUTINE); +typedef BOOL (WINAPI* AcceptExPtr)(SOCKET, SOCKET, PVOID, DWORD, DWORD, DWORD, LPDWORD, LPOVERLAPPED); +typedef BOOL (WINAPI *ConnectExPtr)(SOCKET, const struct sockaddr *, int, PVOID, DWORD, LPDWORD, LPOVERLAPPED); +typedef BOOL (WINAPI *DisconnectExPtr)(SOCKET, LPOVERLAPPED, DWORD, DWORD); typedef struct _GUID { DWORD Data1; diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py --- a/lib_pypy/_pypy_winbase_cffi.py +++ b/lib_pypy/_pypy_winbase_cffi.py @@ -3,8 +3,8 @@ ffi = _cffi_backend.FFI('_pypy_winbase_cffi', _version = 0x2601, - _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x23\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x13\x03\x00\x01\x13\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x1A\x03\x00\x00\x1A\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x1A\x11\x00\x00\x1A\x11\x00\x01\x18\x03\x00\x01\x14\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x03\x00\x00\x26\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x26\x11\x00\x00\x11\x11\x00\x01\x0E\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x14\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x19\x03\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x15\x11\x00\x01\x09\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x48\x11\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x4D\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x14\x03\x00\x00\x15\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\x4D\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x26\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x23\x03\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\xC5\x03\x00\x00\x9D\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x9D\x11\x00\x00\x9D\x11\x00\x00\x22\x11\x00\x00\x23\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x5A\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x0A\x01\x00\x00\x26\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x13\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x11\x11\x00\x00\x9D\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x02\x0F\x00\x00\xC0\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\xC0\x0D\x00\x00\x00\x0F\x00\x00\xC0\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x01\x17\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x01\x1A\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xC8\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xC5\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xCB\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xC8\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xCB\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xC8\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x9D\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xC8\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xD1\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xC8\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x01\x23\x0D\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x00\x0F\x00\x01\x23\x0D\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x04\x09\x00\x00\x05\x09\x00\x00\x02\x09\x00\x01\x16\x03\x00\x00\x06\x09\x00\x00\x07\x09\x00\x00\x03\x09\x00\x00\x08\x09\x00\x00\x02\x01\x00\x00\x40\x03\x00\x00\x82\x03\x00\x00\x01\x09\x00\x00\x00\x09\x00\x01\x20\x03\x00\x00\x04\x01\x00\x01\x20\x05\x00\x00\x00\x08\x00\x00\x00\x01', - _globals = (b'\x00\x00\x33\x23CancelIo',0,b'\x00\x00\x36\x23CancelIoEx',0,b'\x00\x00\x33\x23CloseHandle',0,b'\x00\x00\x36\x23ConnectNamedPipe',0,b'\x00\x00\xC7\x23CreateEventA',0,b'\x00\x00\xCD\x23CreateEventW',0,b'\x00\x00\xD3\x23CreateFileA',0,b'\x00\x01\x00\x23CreateFileW',0,b'\x00\x00\xEE\x23CreateIoCompletionPort',0,b'\x00\x00\xDC\x23CreateNamedPipeA',0,b'\x00\x00\xF6\x23CreateNamedPipeW',0,b'\x00\x00\x25\x23CreatePipe',0,b'\x00\x00\x19\x23CreateProcessA',0,b'\x00\x00\x9C\x23CreateProcessW',0,b'\x00\x00\x8C\x23DuplicateHandle',0,b'\x00\x00\xF4\x23GetCurrentProcess',0,b'\x00\x00\x5C\x23GetExitCodeProcess',0,b'\x00\x00\xBD\x23GetLastError',0,b'\x00\x00\xB8\x23GetModuleFileNameW',0,b'\x00\x00\x3A\x23GetOverlappedResult',0,b'\x00\x00\x60\x23GetQueuedCompletionStatus',0,b'\x00\x00\xEB\x23GetStdHandle',0,b'\x00\x00\xBD\x23GetVersion',0,b'\x00\x00\x6D\x23PostQueuedCompletionStatus',0,b'\x00\x00\x10\x23ReadFile',0,b'\x00\x00\x2B\x23RegisterWaitForSingleObject',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\xAB\x23SetErrorMode',0,b'\x00\x00\x33\x23SetEvent',0,b'\x00\x00\x67\x23SetNamedPipeHandleState',0,b'\x00\x00\x58\x23TerminateProcess',0,b'\x00\x00\x33\x23UnregisterWait',0,b'\x00\x00\x7E\x23UnregisterWaitEx',0,b'\x00\x00\x73\x23WSAIoctl',0,b'\x00\x00\x46\x23WSARecv',0,b'\x00\x00\x4F\x23WSASend',0,b'\xFF\xFF\xFF\x1FWT_EXECUTEINWAITTHREAD',4,b'\xFF\xFF\xFF\x1FWT_EXECUTEONLYONCE',8,b'\x00\x00\xAE\x23WaitForMultipleObjects',0,b'\x00\x00\xB4\x23WaitForSingleObject',0,b'\x00\x00\x95\x23WriteFile',0,b'\x00\x00\xA8\x23_get_osfhandle',0,b'\x00\x00\x17\x23_getch',0,b'\x00\x00\x17\x23_getche',0,b'\x00\x00\xC2\x23_getwch',0,b'\x00\x00\xC2\x23_getwche',0,b'\x00\x00\x17\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\xC4\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\xBF\x23_ungetwch',0,b'\x00\x00\xE6\x23socket',0), - _struct_unions = ((b'\x00\x00\x01\x1E\x00\x00\x00\x03$1',b'\x00\x01\x1D\x11DUMMYSTRUCTNAME',b'\x00\x00\x11\x11Pointer'),(b'\x00\x00\x01\x1D\x00\x00\x00\x02$2',b'\x00\x00\x13\x11Offset',b'\x00\x00\x13\x11OffsetHigh'),(b'\x00\x00\x01\x14\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x11\x11hProcess',b'\x00\x00\x11\x11hThread',b'\x00\x00\x13\x11dwProcessId',b'\x00\x00\x13\x11dwThreadId'),(b'\x00\x00\x01\x18\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x13\x11cb',b'\x00\x00\x1A\x11lpReserved',b'\x00\x00\x1A\x11lpDesktop',b'\x00\x00\x1A\x11lpTitle',b'\x00\x00\x13\x11dwX',b'\x00\x00\x13\x11dwY',b'\x00\x00\x13\x11dwXSize',b'\x00\x00\x13\x11dwYSize',b'\x00\x00\x13\x11dwXCountChars',b'\x00\x00\x13\x11dwYCountChars',b'\x00\x00\x13\x11dwFillAttribute',b'\x00\x00\x13\x11dwFlags',b'\x00\x00\xC0\x11wShowWindow',b'\x00\x00\xC0\x11cbReserved2',b'\x00\x01\x1F\x11lpReserved2',b'\x00\x00\x11\x11hStdInput',b'\x00\x00\x11\x11hStdOutput',b'\x00\x00\x11\x11hStdError'),(b'\x00\x00\x01\x12\x00\x00\x00\x02_GUID',b'\x00\x00\x13\x11Data1',b'\x00\x00\xC0\x11Data2',b'\x00\x00\xC0\x11Data3',b'\x00\x01\x21\x11Data4'),(b'\x00\x00\x01\x13\x00\x00\x00\x02_OVERLAPPED',b'\x00\x00\x13\x11Internal',b'\x00\x00\x13\x11InternalHigh',b'\x00\x01\x1E\x11DUMMYUNIONNAME',b'\x00\x00\x11\x11hEvent'),(b'\x00\x00\x01\x16\x00\x00\x00\x02_PostCallbackData',b'\x00\x00\x11\x11hCompletionPort',b'\x00\x00\x15\x11Overlapped'),(b'\x00\x00\x01\x17\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x13\x11nLength',b'\x00\x00\x11\x11lpSecurityDescriptor',b'\x00\x00\x01\x11bInheritHandle'),(b'\x00\x00\x01\x19\x00\x00\x00\x02_WSABUF',b'\x00\x00\x13\x11len',b'\x00\x00\x1A\x11buf')), - _typenames = (b'\x00\x00\x00\x82AcceptEx',b'\x00\x00\x01\x1CAcceptExPtr',b'\x00\x00\x01\x12GUID',b'\x00\x00\x01\x1BLPFN_DISCONNECTEX',b'\x00\x00\x00\x15LPOVERLAPPED',b'\x00\x00\x00\x4DLPOVERLAPPED_COMPLETION_ROUTINE',b'\x00\x00\x00\x23LPPROCESS_INFORMATION',b'\x00\x00\x01\x15LPPostCallbackData',b'\x00\x00\x00\xC8LPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x22LPSTARTUPINFO',b'\x00\x00\x00\x48LPWSABUF',b'\x00\x00\x01\x13OVERLAPPED',b'\x00\x00\x01\x14PROCESS_INFORMATION',b'\x00\x00\x00\xC8PSECURITY_ATTRIBUTES',b'\x00\x00\x01\x16PostCallbackData',b'\x00\x00\x01\x17SECURITY_ATTRIBUTES',b'\x00\x00\x00\x11SOCKET',b'\x00\x00\x01\x18STARTUPINFO',b'\x00\x00\x00\x2EWAITORTIMERCALLBACK',b'\x00\x00\x01\x19WSABUF',b'\x00\x00\x00\xC0wint_t'), + _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x2E\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x13\x03\x00\x01\x1C\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x23\x03\x00\x00\x1A\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x1A\x11\x00\x00\x1A\x11\x00\x01\x21\x03\x00\x01\x1D\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x03\x00\x00\x26\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x26\x11\x00\x00\x11\x11\x00\x01\x17\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x14\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x22\x03\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x15\x11\x00\x01\x12\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x48\x11\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x4D\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x28\x03\x00\x00\x07\x01\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x14\x03\x00\x00\x15\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\x4D\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x26\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x2E\x03\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\xCE\x03\x00\x00\xA6\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\xA6\x11\x00\x00\xA6\x11\x00\x00\x22\x11\x00\x00\x23\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x63\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x0A\x01\x00\x00\x26\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x13\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x11\x11\x00\x00\xA6\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x02\x0F\x00\x00\xC9\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\xC9\x0D\x00\x00\x00\x0F\x00\x00\xC9\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x01\x20\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x01\x23\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xD1\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xCE\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xD4\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xD1\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xD4\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xD1\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xA6\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xD1\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xDA\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xD1\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x01\x2E\x0D\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x00\x0F\x00\x01\x2E\x0D\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x04\x09\x00\x00\x05\x09\x00\x00\x02\x09\x00\x01\x1F\x03\x00\x00\x06\x09\x00\x00\x07\x09\x00\x00\x03\x09\x00\x00\x08\x09\x00\x00\x02\x01\x00\x00\x40\x03\x00\x00\x58\x03\x00\x00\x8B\x03\x00\x00\x01\x09\x00\x00\x09\x09\x00\x00\x00\x09\x00\x01\x2B\x03\x00\x00\x04\x01\x00\x01\x2B\x05\x00\x00\x00\x08\x00\x00\x00\x01', + _globals = (b'\x00\x00\x33\x23CancelIo',0,b'\x00\x00\x36\x23CancelIoEx',0,b'\x00\x00\x33\x23CloseHandle',0,b'\x00\x00\x36\x23ConnectNamedPipe',0,b'\x00\x00\xD0\x23CreateEventA',0,b'\x00\x00\xD6\x23CreateEventW',0,b'\x00\x00\xDC\x23CreateFileA',0,b'\x00\x01\x09\x23CreateFileW',0,b'\x00\x00\xF7\x23CreateIoCompletionPort',0,b'\x00\x00\xE5\x23CreateNamedPipeA',0,b'\x00\x00\xFF\x23CreateNamedPipeW',0,b'\x00\x00\x25\x23CreatePipe',0,b'\x00\x00\x19\x23CreateProcessA',0,b'\x00\x00\xA5\x23CreateProcessW',0,b'\x00\x00\x95\x23DuplicateHandle',0,b'\x00\x00\xFD\x23GetCurrentProcess',0,b'\x00\x00\x65\x23GetExitCodeProcess',0,b'\x00\x00\xC6\x23GetLastError',0,b'\x00\x00\xC1\x23GetModuleFileNameW',0,b'\x00\x00\x3A\x23GetOverlappedResult',0,b'\x00\x00\x69\x23GetQueuedCompletionStatus',0,b'\x00\x00\xF4\x23GetStdHandle',0,b'\x00\x00\xC6\x23GetVersion',0,b'\x00\x00\x76\x23PostQueuedCompletionStatus',0,b'\x00\x00\x10\x23ReadFile',0,b'\x00\x00\x2B\x23RegisterWaitForSingleObject',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\xB4\x23SetErrorMode',0,b'\x00\x00\x33\x23SetEvent',0,b'\x00\x00\x70\x23SetNamedPipeHandleState',0,b'\x00\x00\x61\x23TerminateProcess',0,b'\x00\x00\x33\x23UnregisterWait',0,b'\x00\x00\x87\x23UnregisterWaitEx',0,b'\x00\x00\x7C\x23WSAIoctl',0,b'\x00\x00\x46\x23WSARecv',0,b'\x00\x00\x4F\x23WSASend',0,b'\xFF\xFF\xFF\x1FWT_EXECUTEINWAITTHREAD',4,b'\xFF\xFF\xFF\x1FWT_EXECUTEONLYONCE',8,b'\x00\x00\xB7\x23WaitForMultipleObjects',0,b'\x00\x00\xBD\x23WaitForSingleObject',0,b'\x00\x00\x9E\x23WriteFile',0,b'\x00\x00\xB1\x23_get_osfhandle',0,b'\x00\x00\x17\x23_getch',0,b'\x00\x00\x17\x23_getche',0,b'\x00\x00\xCB\x23_getwch',0,b'\x00\x00\xCB\x23_getwche',0,b'\x00\x00\x17\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\xCD\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\xC8\x23_ungetwch',0,b'\x00\x00\xEF\x23socket',0), + _struct_unions = ((b'\x00\x00\x01\x29\x00\x00\x00\x03$1',b'\x00\x01\x27\x11DUMMYSTRUCTNAME',b'\x00\x00\x11\x11Pointer'),(b'\x00\x00\x01\x27\x00\x00\x00\x02$2',b'\x00\x00\x13\x11Offset',b'\x00\x00\x13\x11OffsetHigh'),(b'\x00\x00\x01\x1D\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x11\x11hProcess',b'\x00\x00\x11\x11hThread',b'\x00\x00\x13\x11dwProcessId',b'\x00\x00\x13\x11dwThreadId'),(b'\x00\x00\x01\x21\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x13\x11cb',b'\x00\x00\x1A\x11lpReserved',b'\x00\x00\x1A\x11lpDesktop',b'\x00\x00\x1A\x11lpTitle',b'\x00\x00\x13\x11dwX',b'\x00\x00\x13\x11dwY',b'\x00\x00\x13\x11dwXSize',b'\x00\x00\x13\x11dwYSize',b'\x00\x00\x13\x11dwXCountChars',b'\x00\x00\x13\x11dwYCountChars',b'\x00\x00\x13\x11dwFillAttribute',b'\x00\x00\x13\x11dwFlags',b'\x00\x00\xC9\x11wShowWindow',b'\x00\x00\xC9\x11cbReserved2',b'\x00\x01\x2A\x11lpReserved2',b'\x00\x00\x11\x11hStdInput',b'\x00\x00\x11\x11hStdOutput',b'\x00\x00\x11\x11hStdError'),(b'\x00\x00\x01\x1B\x00\x00\x00\x02_GUID',b'\x00\x00\x13\x11Data1',b'\x00\x00\xC9\x11Data2',b'\x00\x00\xC9\x11Data3',b'\x00\x01\x2C\x11Data4'),(b'\x00\x00\x01\x1C\x00\x00\x00\x02_OVERLAPPED',b'\x00\x00\x13\x11Internal',b'\x00\x00\x13\x11InternalHigh',b'\x00\x01\x29\x11DUMMYUNIONNAME',b'\x00\x00\x11\x11hEvent'),(b'\x00\x00\x01\x1F\x00\x00\x00\x02_PostCallbackData',b'\x00\x00\x11\x11hCompletionPort',b'\x00\x00\x15\x11Overlapped'),(b'\x00\x00\x01\x20\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x13\x11nLength',b'\x00\x00\x11\x11lpSecurityDescriptor',b'\x00\x00\x01\x11bInheritHandle'),(b'\x00\x00\x01\x22\x00\x00\x00\x02_WSABUF',b'\x00\x00\x13\x11len',b'\x00\x00\x1A\x11buf'),(b'\x00\x00\x01\x28\x00\x00\x00\x10sockaddr',)), + _typenames = (b'\x00\x00\x01\x26AcceptExPtr',b'\x00\x00\x01\x25ConnectExPtr',b'\x00\x00\x01\x24DisconnectExPtr',b'\x00\x00\x01\x1BGUID',b'\x00\x00\x01\x24LPFN_DISCONNECTEX',b'\x00\x00\x00\x15LPOVERLAPPED',b'\x00\x00\x00\x4DLPOVERLAPPED_COMPLETION_ROUTINE',b'\x00\x00\x00\x23LPPROCESS_INFORMATION',b'\x00\x00\x01\x1ELPPostCallbackData',b'\x00\x00\x00\xD1LPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x22LPSTARTUPINFO',b'\x00\x00\x00\x48LPWSABUF',b'\x00\x00\x01\x1COVERLAPPED',b'\x00\x00\x01\x1DPROCESS_INFORMATION',b'\x00\x00\x00\xD1PSECURITY_ATTRIBUTES',b'\x00\x00\x01\x1FPostCallbackData',b'\x00\x00\x01\x20SECURITY_ATTRIBUTES',b'\x00\x00\x00\x11SOCKET',b'\x00\x00\x01\x21STARTUPINFO',b'\x00\x00\x00\x2EWAITORTIMERCALLBACK',b'\x00\x00\x01\x22WSABUF',b'\x00\x00\x00\xC9wint_t'), ) From pypy.commits at gmail.com Sun Mar 17 11:13:41 2019 From: pypy.commits at gmail.com (andrewjlawrence) Date: Sun, 17 Mar 2019 08:13:41 -0700 (PDT) Subject: [pypy-commit] pypy winoverlapped: Implemented Overlapped.AcceptEx Message-ID: <5c8e6425.1c69fb81.7b71d.7b42@mx.google.com> Author: andrewjlawrence Branch: winoverlapped Changeset: r96333:619d115d9992 Date: 2019-03-17 15:11 +0000 http://bitbucket.org/pypy/pypy/changeset/619d115d9992/ Log: Implemented Overlapped.AcceptEx diff --git a/lib_pypy/_overlapped.py b/lib_pypy/_overlapped.py --- a/lib_pypy/_overlapped.py +++ b/lib_pypy/_overlapped.py @@ -115,17 +115,21 @@ WSAID_ACCEPTEX, _ffi.sizeof(WSAID_ACCEPTEX[0]), _accept_ex, \ _ffi.sizeof(_accept_ex[0]), dwBytes, _ffi.NULL, _ffi.NULL) if result == INVALID_SOCKET: + _winsock2.closesocket(s) raise _winapi._WinError() result = _winsock2.WSAIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, \ WSAID_CONNECTEX, _ffi.sizeof(WSAID_CONNECTEX[0]), _connect_ex, \ _ffi.sizeof(_connect_ex[0]), dwBytes, _ffi.NULL, _ffi.NULL) if result == INVALID_SOCKET: + _winsock2.closesocket(s) raise _winapi._WinError() result = _winsock2.WSAIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, \ WSAID_DISCONNECTEX, _ffi.sizeof(WSAID_DISCONNECTEX[0]), _disconnect_ex, \ _ffi.sizeof(_disconnect_ex[0]), dwBytes, _ffi.NULL, _ffi.NULL) + + _winsock2.closesocket(s) if result == INVALID_SOCKET: raise _winapi._WinError() @@ -317,8 +321,9 @@ err = _winapi.ERROR_SUCCESS else: err = _kernel32.GetLastError() - self.error = err - + + self.error = err + if err == _winapi.ERROR_BROKEN_PIPE: mark_as_completed(self.overlapped) raise _winapi._WinError() @@ -342,19 +347,52 @@ ret = _kernel32.WriteFile(self.handle, self.write_buffer, len(self.write_buffer), written, self.overlapped) if ret: - err = _winapi.ERROR_SUCCESS + self.error = _winapi.ERROR_SUCCESS else: - err = _winapi.GetLastError() + self.error = _winapi.GetLastError() - self.error = err - - if err == _winapi.ERROR_SUCCESS or err == _winapi.ERROR_IO_PENDING: + if self.error == _winapi.ERROR_SUCCESS or self.error == _winapi.ERROR_IO_PENDING: return None else: self.type = OverlappedType.TYPE_NOT_STARTED raise _winapi.WinError() def AcceptEx(self, listensocket, acceptsocket): + listensocket = _int2handle(listensocket) + acceptsocket = _int2handle(acceptsocket) + bytesreceived = _ffi.new("DWORD[1]") + + if self.type != OverlappedType.TYPE_NONE: + raise _winapi._WinError() + + size = _ffi.sizeof("struct sockaddr_in6") + 16 + buf = _ffi.new("CHAR[]", size*2) + if not buf: + return None + + self.type = OverlappedType.TYPE_ACCEPT + self.handle = listensocket + self.read_buffer = buf + + res = _accept_ex[0](listensocket, acceptsocket, buf, \ + 0, size, size, bytesreceived, self.overlapped) + + if res: + self.error = _winapi.ERROR_SUCCESS + else: + self.error = _kernel32.GetLastError() + + if self.error == _winapi.ERROR_SUCCESS or self.error == _winapi.ERROR_IO_PENDING: + return None + else: + self.type = OverlappedType.TYPE_NOT_STARTED + raise _winapi.WinError() + + def DisconnectEx(self, socket, flags): + xxx + return None + + def ConnectEx(self, socket, flags): xxx return None diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py --- a/lib_pypy/_pypy_winbase_build.py +++ b/lib_pypy/_pypy_winbase_build.py @@ -164,6 +164,7 @@ typedef HANDLE SOCKET; SOCKET __stdcall socket(int, int, int); +int closesocket(SOCKET); typedef BOOL (__stdcall * LPFN_DISCONNECTEX) (SOCKET, LPOVERLAPPED, DWORD, DWORD); typedef VOID (*LPOVERLAPPED_COMPLETION_ROUTINE) (DWORD, DWORD, LPVOID); @@ -182,6 +183,36 @@ BYTE Data4[8]; } GUID; +typedef USHORT ADDRESS_FAMILY; + +typedef struct in6_addr { + union { + UCHAR Byte[16]; + USHORT Word[8]; + } u; +} IN6_ADDR, *PIN6_ADDR, *LPIN6_ADDR; + +typedef struct { + union { + struct { + ULONG Zone : 28; + ULONG Level : 4; + }; + ULONG Value; + }; +} SCOPE_ID, *PSCOPE_ID; + +typedef struct sockaddr_in6 { + ADDRESS_FAMILY sin6_family; + USHORT sin6_port; + ULONG sin6_flowinfo; + IN6_ADDR sin6_addr; + union { + ULONG sin6_scope_id; + SCOPE_ID sin6_scope_struct; + }; +} SOCKADDR_IN6_LH, *PSOCKADDR_IN6_LH, *LPSOCKADDR_IN6_LH; + """) if __name__ == "__main__": diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py --- a/lib_pypy/_pypy_winbase_cffi.py +++ b/lib_pypy/_pypy_winbase_cffi.py @@ -3,8 +3,8 @@ ffi = _cffi_backend.FFI('_pypy_winbase_cffi', _version = 0x2601, - _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x2E\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x13\x03\x00\x01\x1C\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x23\x03\x00\x00\x1A\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x1A\x11\x00\x00\x1A\x11\x00\x01\x21\x03\x00\x01\x1D\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x03\x00\x00\x26\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x26\x11\x00\x00\x11\x11\x00\x01\x17\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x14\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x22\x03\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x15\x11\x00\x01\x12\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x48\x11\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x4D\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x28\x03\x00\x00\x07\x01\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x14\x03\x00\x00\x15\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x14\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\x4D\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x26\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x2E\x03\x00\x00\x0A\x01\x00\x00\x14\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\xCE\x03\x00\x00\xA6\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\xA6\x11\x00\x00\xA6\x11\x00\x00\x22\x11\x00\x00\x23\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x63\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x0A\x01\x00\x00\x26\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x13\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x11\x11\x00\x00\xA6\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x13\x0D\x00\x00\x02\x0F\x00\x00\xC9\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\xC9\x0D\x00\x00\x00\x0F\x00\x00\xC9\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x01\x20\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x01\x23\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xD1\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xCE\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xD4\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xD1\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xD4\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xD1\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xA6\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xD1\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xDA\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xD1\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x01\x2E\x0D\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x00\x0F\x00\x01\x2E\x0D\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x04\x09\x00\x00\x05\x09\x00\x00\x02\x09\x00\x01\x1F\x03\x00\x00\x06\x09\x00\x00\x07\x09\x00\x00\x03\x09\x00\x00\x08\x09\x00\x00\x02\x01\x00\x00\x40\x03\x00\x00\x58\x03\x00\x00\x8B\x03\x00\x00\x01\x09\x00\x00\x09\x09\x00\x00\x00\x09\x00\x01\x2B\x03\x00\x00\x04\x01\x00\x01\x2B\x05\x00\x00\x00\x08\x00\x00\x00\x01', - _globals = (b'\x00\x00\x33\x23CancelIo',0,b'\x00\x00\x36\x23CancelIoEx',0,b'\x00\x00\x33\x23CloseHandle',0,b'\x00\x00\x36\x23ConnectNamedPipe',0,b'\x00\x00\xD0\x23CreateEventA',0,b'\x00\x00\xD6\x23CreateEventW',0,b'\x00\x00\xDC\x23CreateFileA',0,b'\x00\x01\x09\x23CreateFileW',0,b'\x00\x00\xF7\x23CreateIoCompletionPort',0,b'\x00\x00\xE5\x23CreateNamedPipeA',0,b'\x00\x00\xFF\x23CreateNamedPipeW',0,b'\x00\x00\x25\x23CreatePipe',0,b'\x00\x00\x19\x23CreateProcessA',0,b'\x00\x00\xA5\x23CreateProcessW',0,b'\x00\x00\x95\x23DuplicateHandle',0,b'\x00\x00\xFD\x23GetCurrentProcess',0,b'\x00\x00\x65\x23GetExitCodeProcess',0,b'\x00\x00\xC6\x23GetLastError',0,b'\x00\x00\xC1\x23GetModuleFileNameW',0,b'\x00\x00\x3A\x23GetOverlappedResult',0,b'\x00\x00\x69\x23GetQueuedCompletionStatus',0,b'\x00\x00\xF4\x23GetStdHandle',0,b'\x00\x00\xC6\x23GetVersion',0,b'\x00\x00\x76\x23PostQueuedCompletionStatus',0,b'\x00\x00\x10\x23ReadFile',0,b'\x00\x00\x2B\x23RegisterWaitForSingleObject',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\xB4\x23SetErrorMode',0,b'\x00\x00\x33\x23SetEvent',0,b'\x00\x00\x70\x23SetNamedPipeHandleState',0,b'\x00\x00\x61\x23TerminateProcess',0,b'\x00\x00\x33\x23UnregisterWait',0,b'\x00\x00\x87\x23UnregisterWaitEx',0,b'\x00\x00\x7C\x23WSAIoctl',0,b'\x00\x00\x46\x23WSARecv',0,b'\x00\x00\x4F\x23WSASend',0,b'\xFF\xFF\xFF\x1FWT_EXECUTEINWAITTHREAD',4,b'\xFF\xFF\xFF\x1FWT_EXECUTEONLYONCE',8,b'\x00\x00\xB7\x23WaitForMultipleObjects',0,b'\x00\x00\xBD\x23WaitForSingleObject',0,b'\x00\x00\x9E\x23WriteFile',0,b'\x00\x00\xB1\x23_get_osfhandle',0,b'\x00\x00\x17\x23_getch',0,b'\x00\x00\x17\x23_getche',0,b'\x00\x00\xCB\x23_getwch',0,b'\x00\x00\xCB\x23_getwche',0,b'\x00\x00\x17\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\xCD\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\xC8\x23_ungetwch',0,b'\x00\x00\xEF\x23socket',0), - _struct_unions = ((b'\x00\x00\x01\x29\x00\x00\x00\x03$1',b'\x00\x01\x27\x11DUMMYSTRUCTNAME',b'\x00\x00\x11\x11Pointer'),(b'\x00\x00\x01\x27\x00\x00\x00\x02$2',b'\x00\x00\x13\x11Offset',b'\x00\x00\x13\x11OffsetHigh'),(b'\x00\x00\x01\x1D\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x11\x11hProcess',b'\x00\x00\x11\x11hThread',b'\x00\x00\x13\x11dwProcessId',b'\x00\x00\x13\x11dwThreadId'),(b'\x00\x00\x01\x21\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x13\x11cb',b'\x00\x00\x1A\x11lpReserved',b'\x00\x00\x1A\x11lpDesktop',b'\x00\x00\x1A\x11lpTitle',b'\x00\x00\x13\x11dwX',b'\x00\x00\x13\x11dwY',b'\x00\x00\x13\x11dwXSize',b'\x00\x00\x13\x11dwYSize',b'\x00\x00\x13\x11dwXCountChars',b'\x00\x00\x13\x11dwYCountChars',b'\x00\x00\x13\x11dwFillAttribute',b'\x00\x00\x13\x11dwFlags',b'\x00\x00\xC9\x11wShowWindow',b'\x00\x00\xC9\x11cbReserved2',b'\x00\x01\x2A\x11lpReserved2',b'\x00\x00\x11\x11hStdInput',b'\x00\x00\x11\x11hStdOutput',b'\x00\x00\x11\x11hStdError'),(b'\x00\x00\x01\x1B\x00\x00\x00\x02_GUID',b'\x00\x00\x13\x11Data1',b'\x00\x00\xC9\x11Data2',b'\x00\x00\xC9\x11Data3',b'\x00\x01\x2C\x11Data4'),(b'\x00\x00\x01\x1C\x00\x00\x00\x02_OVERLAPPED',b'\x00\x00\x13\x11Internal',b'\x00\x00\x13\x11InternalHigh',b'\x00\x01\x29\x11DUMMYUNIONNAME',b'\x00\x00\x11\x11hEvent'),(b'\x00\x00\x01\x1F\x00\x00\x00\x02_PostCallbackData',b'\x00\x00\x11\x11hCompletionPort',b'\x00\x00\x15\x11Overlapped'),(b'\x00\x00\x01\x20\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x13\x11nLength',b'\x00\x00\x11\x11lpSecurityDescriptor',b'\x00\x00\x01\x11bInheritHandle'),(b'\x00\x00\x01\x22\x00\x00\x00\x02_WSABUF',b'\x00\x00\x13\x11len',b'\x00\x00\x1A\x11buf'),(b'\x00\x00\x01\x28\x00\x00\x00\x10sockaddr',)), - _typenames = (b'\x00\x00\x01\x26AcceptExPtr',b'\x00\x00\x01\x25ConnectExPtr',b'\x00\x00\x01\x24DisconnectExPtr',b'\x00\x00\x01\x1BGUID',b'\x00\x00\x01\x24LPFN_DISCONNECTEX',b'\x00\x00\x00\x15LPOVERLAPPED',b'\x00\x00\x00\x4DLPOVERLAPPED_COMPLETION_ROUTINE',b'\x00\x00\x00\x23LPPROCESS_INFORMATION',b'\x00\x00\x01\x1ELPPostCallbackData',b'\x00\x00\x00\xD1LPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x22LPSTARTUPINFO',b'\x00\x00\x00\x48LPWSABUF',b'\x00\x00\x01\x1COVERLAPPED',b'\x00\x00\x01\x1DPROCESS_INFORMATION',b'\x00\x00\x00\xD1PSECURITY_ATTRIBUTES',b'\x00\x00\x01\x1FPostCallbackData',b'\x00\x00\x01\x20SECURITY_ATTRIBUTES',b'\x00\x00\x00\x11SOCKET',b'\x00\x00\x01\x21STARTUPINFO',b'\x00\x00\x00\x2EWAITORTIMERCALLBACK',b'\x00\x00\x01\x22WSABUF',b'\x00\x00\x00\xC9wint_t'), + _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x3F\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x16\x03\x00\x01\x21\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x2C\x03\x00\x00\x1D\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x1D\x11\x00\x00\x1D\x11\x00\x01\x2A\x03\x00\x01\x22\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x03\x00\x00\x29\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x29\x11\x00\x00\x11\x11\x00\x01\x1A\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x18\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x18\x11\x00\x00\x17\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x18\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x2B\x03\x00\x00\x0A\x01\x00\x00\x17\x11\x00\x00\x17\x11\x00\x00\x18\x11\x00\x01\x15\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x4B\x11\x00\x00\x0A\x01\x00\x00\x17\x11\x00\x00\x0A\x01\x00\x00\x18\x11\x00\x00\x50\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x32\x03\x00\x00\x07\x01\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x17\x11\x00\x00\x18\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x17\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x17\x11\x00\x00\x17\x03\x00\x00\x18\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x17\x11\x00\x00\x17\x11\x00\x00\x17\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x18\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x17\x11\x00\x00\x18\x11\x00\x00\x50\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x17\x11\x00\x00\x18\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x29\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x3F\x03\x00\x00\x0A\x01\x00\x00\x17\x11\x00\x00\x18\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\xD1\x03\x00\x00\xA9\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\xA9\x11\x00\x00\xA9\x11\x00\x00\x25\x11\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x66\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x16\x0D\x00\x00\x0A\x01\x00\x00\x29\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x16\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x16\x0D\x00\x00\x11\x11\x00\x00\xA9\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x16\x0D\x00\x00\x02\x0F\x00\x00\xCC\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\xCC\x0D\x00\x00\x00\x0F\x00\x00\xCC\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x01\x27\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x01\x2C\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xD4\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xD1\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xD7\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xD4\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xD7\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xD4\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xA9\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xD4\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xDD\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xD4\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x01\x3F\x0D\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x00\x0F\x00\x01\x3F\x0D\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x09\x09\x00\x01\x20\x03\x00\x00\x0E\x09\x00\x00\x0A\x09\x00\x00\x06\x09\x00\x01\x24\x03\x00\x00\x0B\x09\x00\x01\x26\x03\x00\x00\x07\x09\x00\x00\x0C\x09\x00\x01\x29\x03\x00\x00\x10\x09\x00\x00\x08\x09\x00\x00\x0D\x09\x00\x00\x02\x01\x00\x00\x43\x03\x00\x00\x5B\x03\x00\x00\x8E\x03\x00\x00\x01\x09\x00\x00\x04\x09\x00\x00\x0F\x09\x00\x00\x00\x09\x00\x00\x02\x09\x00\x00\x03\x09\x00\x00\x05\x09\x00\x01\x38\x03\x00\x00\x04\x01\x00\x01\x38\x05\x00\x00\x00\x10\x00\x01\x38\x05\x00\x00\x00\x08\x00\x00\xCC\x05\x00\x00\x00\x08\x00\x00\x00\x01', + _globals = (b'\x00\x00\x36\x23CancelIo',0,b'\x00\x00\x39\x23CancelIoEx',0,b'\x00\x00\x36\x23CloseHandle',0,b'\x00\x00\x39\x23ConnectNamedPipe',0,b'\x00\x00\xD3\x23CreateEventA',0,b'\x00\x00\xD9\x23CreateEventW',0,b'\x00\x00\xDF\x23CreateFileA',0,b'\x00\x01\x0C\x23CreateFileW',0,b'\x00\x00\xFA\x23CreateIoCompletionPort',0,b'\x00\x00\xE8\x23CreateNamedPipeA',0,b'\x00\x01\x02\x23CreateNamedPipeW',0,b'\x00\x00\x28\x23CreatePipe',0,b'\x00\x00\x1C\x23CreateProcessA',0,b'\x00\x00\xA8\x23CreateProcessW',0,b'\x00\x00\x98\x23DuplicateHandle',0,b'\x00\x01\x00\x23GetCurrentProcess',0,b'\x00\x00\x68\x23GetExitCodeProcess',0,b'\x00\x00\xC9\x23GetLastError',0,b'\x00\x00\xC4\x23GetModuleFileNameW',0,b'\x00\x00\x3D\x23GetOverlappedResult',0,b'\x00\x00\x6C\x23GetQueuedCompletionStatus',0,b'\x00\x00\xF7\x23GetStdHandle',0,b'\x00\x00\xC9\x23GetVersion',0,b'\x00\x00\x79\x23PostQueuedCompletionStatus',0,b'\x00\x00\x13\x23ReadFile',0,b'\x00\x00\x2E\x23RegisterWaitForSingleObject',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\xB7\x23SetErrorMode',0,b'\x00\x00\x36\x23SetEvent',0,b'\x00\x00\x73\x23SetNamedPipeHandleState',0,b'\x00\x00\x64\x23TerminateProcess',0,b'\x00\x00\x36\x23UnregisterWait',0,b'\x00\x00\x8A\x23UnregisterWaitEx',0,b'\x00\x00\x7F\x23WSAIoctl',0,b'\x00\x00\x49\x23WSARecv',0,b'\x00\x00\x52\x23WSASend',0,b'\xFF\xFF\xFF\x1FWT_EXECUTEINWAITTHREAD',4,b'\xFF\xFF\xFF\x1FWT_EXECUTEONLYONCE',8,b'\x00\x00\xBA\x23WaitForMultipleObjects',0,b'\x00\x00\xC0\x23WaitForSingleObject',0,b'\x00\x00\xA1\x23WriteFile',0,b'\x00\x00\xB4\x23_get_osfhandle',0,b'\x00\x00\x1A\x23_getch',0,b'\x00\x00\x1A\x23_getche',0,b'\x00\x00\xCE\x23_getwch',0,b'\x00\x00\xCE\x23_getwche',0,b'\x00\x00\x1A\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\xD0\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\xCB\x23_ungetwch',0,b'\x00\x00\x10\x23closesocket',0,b'\x00\x00\xF2\x23socket',0), + _struct_unions = ((b'\x00\x00\x01\x33\x00\x00\x00\x03$1',b'\x00\x01\x30\x11DUMMYSTRUCTNAME',b'\x00\x00\x11\x11Pointer'),(b'\x00\x00\x01\x30\x00\x00\x00\x02$2',b'\x00\x00\x16\x11Offset',b'\x00\x00\x16\x11OffsetHigh'),(b'\x00\x00\x01\x34\x00\x00\x00\x03$3',b'\x00\x01\x39\x11Byte',b'\x00\x01\x3D\x11Word'),(b'\x00\x00\x01\x35\x00\x00\x00\x01$4',b'\x00\x01\x31\x11',b'\x00\x00\x16\x11Value'),(b'\x00\x00\x01\x31\x00\x00\x00\x02$5',b'\x00\x00\x16\x13\x00\x00\x00\x1CZone',b'\x00\x00\x16\x13\x00\x00\x00\x04Level'),(b'\x00\x00\x01\x36\x00\x00\x00\x03$6',b'\x00\x00\x16\x11sin6_scope_id',b'\x00\x01\x26\x11sin6_scope_struct'),(b'\x00\x00\x01\x22\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x11\x11hProcess',b'\x00\x00\x11\x11hThread',b'\x00\x00\x16\x11dwProcessId',b'\x00\x00\x16\x11dwThreadId'),(b'\x00\x00\x01\x26\x00\x00\x00\x00$SCOPE_ID',b'\x00\x01\x35\x11'),(b'\x00\x00\x01\x2A\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x16\x11cb',b'\x00\x00\x1D\x11lpReserved',b'\x00\x00\x1D\x11lpDesktop',b'\x00\x00\x1D\x11lpTitle',b'\x00\x00\x16\x11dwX',b'\x00\x00\x16\x11dwY',b'\x00\x00\x16\x11dwXSize',b'\x00\x00\x16\x11dwYSize',b'\x00\x00\x16\x11dwXCountChars',b'\x00\x00\x16\x11dwYCountChars',b'\x00\x00\x16\x11dwFillAttribute',b'\x00\x00\x16\x11dwFlags',b'\x00\x00\xCC\x11wShowWindow',b'\x00\x00\xCC\x11cbReserved2',b'\x00\x01\x37\x11lpReserved2',b'\x00\x00\x11\x11hStdInput',b'\x00\x00\x11\x11hStdOutput',b'\x00\x00\x11\x11hStdError'),(b'\x00\x00\x01\x1E\x00\x00\x00\x02_GUID',b'\x00\x00\x16\x11Data1',b'\x00\x00\xCC\x11Data2',b'\x00\x00\xCC\x11Data3',b'\x00\x01\x3B\x11Data4'),(b'\x00\x00\x01\x21\x00\x00\x00\x02_OVERLAPPED',b'\x00\x00\x16\x11Internal',b'\x00\x00\x16\x11InternalHigh',b'\x00\x01\x33\x11DUMMYUNIONNAME',b'\x00\x00\x11\x11hEvent'),(b'\x00\x00\x01\x24\x00\x00\x00\x02_PostCallbackData',b'\x00\x00\x11\x11hCompletionPort',b'\x00\x00\x18\x11Overlapped'),(b'\x00\x00\x01\x27\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x16\x11nLength',b'\x00\x00\x11\x11lpSecurityDescriptor',b'\x00\x00\x01\x11bInheritHandle'),(b'\x00\x00\x01\x2B\x00\x00\x00\x02_WSABUF',b'\x00\x00\x16\x11len',b'\x00\x00\x1D\x11buf'),(b'\x00\x00\x01\x20\x00\x00\x00\x02in6_addr',b'\x00\x01\x34\x11u'),(b'\x00\x00\x01\x32\x00\x00\x00\x10sockaddr',),(b'\x00\x00\x01\x29\x00\x00\x00\x00sockaddr_in6',b'\x00\x00\xCC\x11sin6_family',b'\x00\x00\xCC\x11sin6_port',b'\x00\x00\x16\x11sin6_flowinfo',b'\x00\x01\x20\x11sin6_addr',b'\x00\x01\x36\x11')), + _typenames = (b'\x00\x00\x00\xCCADDRESS_FAMILY',b'\x00\x00\x01\x2FAcceptExPtr',b'\x00\x00\x01\x2EConnectExPtr',b'\x00\x00\x01\x2DDisconnectExPtr',b'\x00\x00\x01\x1EGUID',b'\x00\x00\x01\x20IN6_ADDR',b'\x00\x00\x01\x2DLPFN_DISCONNECTEX',b'\x00\x00\x01\x1FLPIN6_ADDR',b'\x00\x00\x00\x18LPOVERLAPPED',b'\x00\x00\x00\x50LPOVERLAPPED_COMPLETION_ROUTINE',b'\x00\x00\x00\x26LPPROCESS_INFORMATION',b'\x00\x00\x01\x23LPPostCallbackData',b'\x00\x00\x00\xD4LPSECURITY_ATTRIBUTES',b'\x00\x00\x01\x28LPSOCKADDR_IN6_LH',b'\x00\x00\x00\x25LPSTARTUPINFO',b'\x00\x00\x00\x4BLPWSABUF',b'\x00\x00\x01\x21OVERLAPPED',b'\x00\x00\x01\x1FPIN6_ADDR',b'\x00\x00\x01\x22PROCESS_INFORMATION',b'\x00\x00\x01\x25PSCOPE_ID',b'\x00\x00\x00\xD4PSECURITY_ATTRIBUTES',b'\x00\x00\x01\x28PSOCKADDR_IN6_LH',b'\x00\x00\x01\x24PostCallbackData',b'\x00\x00\x01\x26SCOPE_ID',b'\x00\x00\x01\x27SECURITY_ATTRIBUTES',b'\x00\x00\x01\x29SOCKADDR_IN6_LH',b'\x00\x00\x00\x11SOCKET',b'\x00\x00\x01\x2ASTARTUPINFO',b'\x00\x00\x00\x31WAITORTIMERCALLBACK',b'\x00\x00\x01\x2BWSABUF',b'\x00\x00\x00\xCCwint_t'), ) From pypy.commits at gmail.com Sun Mar 17 16:25:30 2019 From: pypy.commits at gmail.com (andrewjlawrence) Date: Sun, 17 Mar 2019 13:25:30 -0700 (PDT) Subject: [pypy-commit] pypy winoverlapped: Add BindLocal and ConnectEx Message-ID: <5c8ead3a.1c69fb81.eb589.c87e@mx.google.com> Author: andrewjlawrence Branch: winoverlapped Changeset: r96334:cf8f6013c5f0 Date: 2019-03-17 20:20 +0000 http://bitbucket.org/pypy/pypy/changeset/cf8f6013c5f0/ Log: Add BindLocal and ConnectEx diff --git a/lib_pypy/_overlapped.py b/lib_pypy/_overlapped.py --- a/lib_pypy/_overlapped.py +++ b/lib_pypy/_overlapped.py @@ -33,6 +33,8 @@ SOCKET_ERROR = -1 AF_INET = 2 +AF_INET6 = 23 + SOCK_STREAM = 1 IPPROTO_TCP = 6 @@ -68,6 +70,9 @@ SIO_GET_EXTENSION_FUNCTION_POINTER = _WSAIORW(IOC_WS2,6) SO_UPDATE_ACCEPT_CONTEXT = 0x700B +SO_UPDATE_CONNECT_CONTEXT = 0x7010 +INADDR_ANY = 0x00000000 +in6addr_any = _ffi.new("struct in6_addr[1]") # Status Codes STATUS_PENDING = 0x00000103 @@ -392,9 +397,36 @@ xxx return None - def ConnectEx(self, socket, flags): - xxx - return None + def ConnectEx(self, socket, addressobj): + socket = _int2handle(socket) + + if self.type != OverlappedType.TYPE_NONE: + raise _winapi._WinError() + + address = _ffi.new("struct sockaddr_in6*") + length = _ffi.sizeof("struct sockaddr_in6") + + address, length = parse_address(addressobj, _ffi.cast("SOCKADDR*",address), length) + + if length < 0: + return None + + self.type = OverlappedType.TYPE_CONNECT + self.handle = socket + + res = _connect_ex[0](socket, address, length, \ + _ffi.NULL, 0, _ffi.NULL, self.overlapped) + + if res: + self.error = _winapi.ERROR_SUCCESS + else: + self.error = _kernel32.GetLastError() + + if self.error == _winapi.ERROR_SUCCESS or self.error == _winapi.ERROR_IO_PENDING: + return None + else: + self.type = OverlappedType.TYPE_NOT_STARTED + raise _winapi.WinError() @property def pending(self): @@ -522,14 +554,61 @@ if not ret: raise _winapi._WinError() +def BindLocal(socket, family): + socket = _int2handle(socket) + if family == AF_INET: + addr = _ffi.new("struct sockaddr_in*") + addr[0].sin_family = AF_INET + addr[0].sin_port = 0 + addr[0].sin_addr.S_un.S_addr = INADDR_ANY + paddr = _ffi.cast("PSOCKADDR", addr) + result = _winsock2.bind(socket, paddr, _ffi.sizeof("struct sockaddr_in")) + elif family == AF_INET6: + addr = _ffi.new("struct sockaddr_in6*") + addr.sin6_family = AF_INET6 + addr.sin6_port = 0 + addr.sin6_addr = in6addr_any[0] + result = _winsock2.bind(socket, _ffi.cast("PSOCKADDR", addr), _ffi.sizeof("struct sockaddr_in")) + else: + raise ValueError() + + if result == SOCKET_ERROR: + raise _winapi._WinError() + + # In CPython this function converts a windows error into a python object # Not sure what we should do here. def SetFromWindowsErr(error): return error - def HasOverlappedIoCompleted(overlapped): return (overlapped.Internal != STATUS_PENDING) +def parse_address(addressobj, address, length): + lengthptr = _ffi.new("INT*") + lengthptr[0] = length + if len(addressobj) == 2: + host,port = addressobj + address[0].sa_family = AF_INET + result = _winsock2.WSAStringToAddressW(host, AF_INET, _ffi.NULL, address, lengthptr) + if result < 0: + raise _winapi.WinError() + _ffi.cast("SOCKADDR_IN*",address)[0].sin_port = _winsock2.htons(port) + return address, lengthptr[0] + elif len(addressobj) == 4: + host, port, flowinfo, scopeid = addressobj + address.sa_family = AF_INET6 + result = _winsock2.WSAStringToAddressW(host, AF_INET6, _ffi.NULL, address, lengthptr) + address.sin6_port = _winsock2.htons(port) + address.sin6_flowinfo = flowinfo + address.sin6_scopeid = scopeid + return address, lengthptr[0] + else: + return -1 + + + + + diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py --- a/lib_pypy/_pypy_winbase_build.py +++ b/lib_pypy/_pypy_winbase_build.py @@ -165,6 +165,8 @@ typedef HANDLE SOCKET; SOCKET __stdcall socket(int, int, int); int closesocket(SOCKET); + + typedef BOOL (__stdcall * LPFN_DISCONNECTEX) (SOCKET, LPOVERLAPPED, DWORD, DWORD); typedef VOID (*LPOVERLAPPED_COMPLETION_ROUTINE) (DWORD, DWORD, LPVOID); @@ -172,9 +174,6 @@ int __stdcall WSASend(SOCKET, LPWSABUF, DWORD, LPDWORD, DWORD, LPOVERLAPPED, LPOVERLAPPED_COMPLETION_ROUTINE); int __stdcall WSAIoctl(SOCKET, DWORD, LPVOID, DWORD, LPVOID, DWORD, LPDWORD, LPOVERLAPPED, LPOVERLAPPED_COMPLETION_ROUTINE); -typedef BOOL (WINAPI* AcceptExPtr)(SOCKET, SOCKET, PVOID, DWORD, DWORD, DWORD, LPDWORD, LPOVERLAPPED); -typedef BOOL (WINAPI *ConnectExPtr)(SOCKET, const struct sockaddr *, int, PVOID, DWORD, LPDWORD, LPOVERLAPPED); -typedef BOOL (WINAPI *DisconnectExPtr)(SOCKET, LPOVERLAPPED, DWORD, DWORD); typedef struct _GUID { DWORD Data1; @@ -213,6 +212,75 @@ }; } SOCKADDR_IN6_LH, *PSOCKADDR_IN6_LH, *LPSOCKADDR_IN6_LH; +typedef struct in_addr { + union { + struct { + UCHAR s_b1; + UCHAR s_b2; + UCHAR s_b3; + UCHAR s_b4; + } S_un_b; + struct { + USHORT s_w1; + USHORT s_w2; + } S_un_w; + ULONG S_addr; + } S_un; +} INADDR, *PINADDR; + +typedef struct sockaddr_in { + SHORT sin_family; + USHORT sin_port; + INADDR sin_addr; + CHAR sin_zero[8]; +} SOCKADDR_IN, *PSOCKADDR_IN, *LPSOCKADDR_IN; + +typedef struct sockaddr { + USHORT sa_family; + CHAR sa_data[14]; +} SOCKADDR, *PSOCKADDR, *LPSOCKADDR; + +int bind(SOCKET, const PSOCKADDR, int); + +#define MAX_PROTOCOL_CHAIN 7 + +typedef struct _WSAPROTOCOLCHAIN { + int ChainLen; + DWORD ChainEntries[MAX_PROTOCOL_CHAIN]; +} WSAPROTOCOLCHAIN, *LPWSAPROTOCOLCHAIN; + +#define WSAPROTOCOL_LEN 255 + +typedef struct _WSAPROTOCOL_INFOW { + DWORD dwServiceFlags1; + DWORD dwServiceFlags2; + DWORD dwServiceFlags3; + DWORD dwServiceFlags4; + DWORD dwProviderFlags; + GUID ProviderId; + DWORD dwCatalogEntryId; + WSAPROTOCOLCHAIN ProtocolChain; + int iVersion; + int iAddressFamily; + int iMaxSockAddr; + int iMinSockAddr; + int iSocketType; + int iProtocol; + int iProtocolMaxOffset; + int iNetworkByteOrder; + int iSecurityScheme; + DWORD dwMessageSize; + DWORD dwProviderReserved; + WCHAR szProtocol[WSAPROTOCOL_LEN + 1]; +} WSAPROTOCOL_INFOW, *LPWSAPROTOCOL_INFOW; + +int __stdcall WSAStringToAddressW(LPWSTR, int, LPWSAPROTOCOL_INFOW, LPSOCKADDR, int* ); + +typedef BOOL (WINAPI* AcceptExPtr)(SOCKET, SOCKET, PVOID, DWORD, DWORD, DWORD, LPDWORD, LPOVERLAPPED); +typedef BOOL (WINAPI *ConnectExPtr)(SOCKET, const PSOCKADDR, int, PVOID, DWORD, LPDWORD, LPOVERLAPPED); +typedef BOOL (WINAPI *DisconnectExPtr)(SOCKET, LPOVERLAPPED, DWORD, DWORD); + +USHORT htons(USHORT); """) if __name__ == "__main__": diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py --- a/lib_pypy/_pypy_winbase_cffi.py +++ b/lib_pypy/_pypy_winbase_cffi.py @@ -3,8 +3,8 @@ ffi = _cffi_backend.FFI('_pypy_winbase_cffi', _version = 0x2601, - _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x3F\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x16\x03\x00\x01\x21\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x2C\x03\x00\x00\x1D\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x1D\x11\x00\x00\x1D\x11\x00\x01\x2A\x03\x00\x01\x22\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x03\x00\x00\x29\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x29\x11\x00\x00\x11\x11\x00\x01\x1A\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x18\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x18\x11\x00\x00\x17\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x18\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x2B\x03\x00\x00\x0A\x01\x00\x00\x17\x11\x00\x00\x17\x11\x00\x00\x18\x11\x00\x01\x15\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x4B\x11\x00\x00\x0A\x01\x00\x00\x17\x11\x00\x00\x0A\x01\x00\x00\x18\x11\x00\x00\x50\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x32\x03\x00\x00\x07\x01\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x17\x11\x00\x00\x18\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x17\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x17\x11\x00\x00\x17\x03\x00\x00\x18\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x17\x11\x00\x00\x17\x11\x00\x00\x17\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x18\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x17\x11\x00\x00\x18\x11\x00\x00\x50\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x17\x11\x00\x00\x18\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x29\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x3F\x03\x00\x00\x0A\x01\x00\x00\x17\x11\x00\x00\x18\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\xD1\x03\x00\x00\xA9\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\xA9\x11\x00\x00\xA9\x11\x00\x00\x25\x11\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x66\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x16\x0D\x00\x00\x0A\x01\x00\x00\x29\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x16\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x16\x0D\x00\x00\x11\x11\x00\x00\xA9\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x16\x0D\x00\x00\x02\x0F\x00\x00\xCC\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\xCC\x0D\x00\x00\x00\x0F\x00\x00\xCC\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x01\x27\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x01\x2C\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xD4\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xD1\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xD7\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xD4\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xD7\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xD4\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xA9\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xD4\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xDD\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xD4\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x01\x3F\x0D\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x00\x0F\x00\x01\x3F\x0D\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x09\x09\x00\x01\x20\x03\x00\x00\x0E\x09\x00\x00\x0A\x09\x00\x00\x06\x09\x00\x01\x24\x03\x00\x00\x0B\x09\x00\x01\x26\x03\x00\x00\x07\x09\x00\x00\x0C\x09\x00\x01\x29\x03\x00\x00\x10\x09\x00\x00\x08\x09\x00\x00\x0D\x09\x00\x00\x02\x01\x00\x00\x43\x03\x00\x00\x5B\x03\x00\x00\x8E\x03\x00\x00\x01\x09\x00\x00\x04\x09\x00\x00\x0F\x09\x00\x00\x00\x09\x00\x00\x02\x09\x00\x00\x03\x09\x00\x00\x05\x09\x00\x01\x38\x03\x00\x00\x04\x01\x00\x01\x38\x05\x00\x00\x00\x10\x00\x01\x38\x05\x00\x00\x00\x08\x00\x00\xCC\x05\x00\x00\x00\x08\x00\x00\x00\x01', - _globals = (b'\x00\x00\x36\x23CancelIo',0,b'\x00\x00\x39\x23CancelIoEx',0,b'\x00\x00\x36\x23CloseHandle',0,b'\x00\x00\x39\x23ConnectNamedPipe',0,b'\x00\x00\xD3\x23CreateEventA',0,b'\x00\x00\xD9\x23CreateEventW',0,b'\x00\x00\xDF\x23CreateFileA',0,b'\x00\x01\x0C\x23CreateFileW',0,b'\x00\x00\xFA\x23CreateIoCompletionPort',0,b'\x00\x00\xE8\x23CreateNamedPipeA',0,b'\x00\x01\x02\x23CreateNamedPipeW',0,b'\x00\x00\x28\x23CreatePipe',0,b'\x00\x00\x1C\x23CreateProcessA',0,b'\x00\x00\xA8\x23CreateProcessW',0,b'\x00\x00\x98\x23DuplicateHandle',0,b'\x00\x01\x00\x23GetCurrentProcess',0,b'\x00\x00\x68\x23GetExitCodeProcess',0,b'\x00\x00\xC9\x23GetLastError',0,b'\x00\x00\xC4\x23GetModuleFileNameW',0,b'\x00\x00\x3D\x23GetOverlappedResult',0,b'\x00\x00\x6C\x23GetQueuedCompletionStatus',0,b'\x00\x00\xF7\x23GetStdHandle',0,b'\x00\x00\xC9\x23GetVersion',0,b'\x00\x00\x79\x23PostQueuedCompletionStatus',0,b'\x00\x00\x13\x23ReadFile',0,b'\x00\x00\x2E\x23RegisterWaitForSingleObject',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\xB7\x23SetErrorMode',0,b'\x00\x00\x36\x23SetEvent',0,b'\x00\x00\x73\x23SetNamedPipeHandleState',0,b'\x00\x00\x64\x23TerminateProcess',0,b'\x00\x00\x36\x23UnregisterWait',0,b'\x00\x00\x8A\x23UnregisterWaitEx',0,b'\x00\x00\x7F\x23WSAIoctl',0,b'\x00\x00\x49\x23WSARecv',0,b'\x00\x00\x52\x23WSASend',0,b'\xFF\xFF\xFF\x1FWT_EXECUTEINWAITTHREAD',4,b'\xFF\xFF\xFF\x1FWT_EXECUTEONLYONCE',8,b'\x00\x00\xBA\x23WaitForMultipleObjects',0,b'\x00\x00\xC0\x23WaitForSingleObject',0,b'\x00\x00\xA1\x23WriteFile',0,b'\x00\x00\xB4\x23_get_osfhandle',0,b'\x00\x00\x1A\x23_getch',0,b'\x00\x00\x1A\x23_getche',0,b'\x00\x00\xCE\x23_getwch',0,b'\x00\x00\xCE\x23_getwche',0,b'\x00\x00\x1A\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\xD0\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\xCB\x23_ungetwch',0,b'\x00\x00\x10\x23closesocket',0,b'\x00\x00\xF2\x23socket',0), - _struct_unions = ((b'\x00\x00\x01\x33\x00\x00\x00\x03$1',b'\x00\x01\x30\x11DUMMYSTRUCTNAME',b'\x00\x00\x11\x11Pointer'),(b'\x00\x00\x01\x30\x00\x00\x00\x02$2',b'\x00\x00\x16\x11Offset',b'\x00\x00\x16\x11OffsetHigh'),(b'\x00\x00\x01\x34\x00\x00\x00\x03$3',b'\x00\x01\x39\x11Byte',b'\x00\x01\x3D\x11Word'),(b'\x00\x00\x01\x35\x00\x00\x00\x01$4',b'\x00\x01\x31\x11',b'\x00\x00\x16\x11Value'),(b'\x00\x00\x01\x31\x00\x00\x00\x02$5',b'\x00\x00\x16\x13\x00\x00\x00\x1CZone',b'\x00\x00\x16\x13\x00\x00\x00\x04Level'),(b'\x00\x00\x01\x36\x00\x00\x00\x03$6',b'\x00\x00\x16\x11sin6_scope_id',b'\x00\x01\x26\x11sin6_scope_struct'),(b'\x00\x00\x01\x22\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x11\x11hProcess',b'\x00\x00\x11\x11hThread',b'\x00\x00\x16\x11dwProcessId',b'\x00\x00\x16\x11dwThreadId'),(b'\x00\x00\x01\x26\x00\x00\x00\x00$SCOPE_ID',b'\x00\x01\x35\x11'),(b'\x00\x00\x01\x2A\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x16\x11cb',b'\x00\x00\x1D\x11lpReserved',b'\x00\x00\x1D\x11lpDesktop',b'\x00\x00\x1D\x11lpTitle',b'\x00\x00\x16\x11dwX',b'\x00\x00\x16\x11dwY',b'\x00\x00\x16\x11dwXSize',b'\x00\x00\x16\x11dwYSize',b'\x00\x00\x16\x11dwXCountChars',b'\x00\x00\x16\x11dwYCountChars',b'\x00\x00\x16\x11dwFillAttribute',b'\x00\x00\x16\x11dwFlags',b'\x00\x00\xCC\x11wShowWindow',b'\x00\x00\xCC\x11cbReserved2',b'\x00\x01\x37\x11lpReserved2',b'\x00\x00\x11\x11hStdInput',b'\x00\x00\x11\x11hStdOutput',b'\x00\x00\x11\x11hStdError'),(b'\x00\x00\x01\x1E\x00\x00\x00\x02_GUID',b'\x00\x00\x16\x11Data1',b'\x00\x00\xCC\x11Data2',b'\x00\x00\xCC\x11Data3',b'\x00\x01\x3B\x11Data4'),(b'\x00\x00\x01\x21\x00\x00\x00\x02_OVERLAPPED',b'\x00\x00\x16\x11Internal',b'\x00\x00\x16\x11InternalHigh',b'\x00\x01\x33\x11DUMMYUNIONNAME',b'\x00\x00\x11\x11hEvent'),(b'\x00\x00\x01\x24\x00\x00\x00\x02_PostCallbackData',b'\x00\x00\x11\x11hCompletionPort',b'\x00\x00\x18\x11Overlapped'),(b'\x00\x00\x01\x27\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x16\x11nLength',b'\x00\x00\x11\x11lpSecurityDescriptor',b'\x00\x00\x01\x11bInheritHandle'),(b'\x00\x00\x01\x2B\x00\x00\x00\x02_WSABUF',b'\x00\x00\x16\x11len',b'\x00\x00\x1D\x11buf'),(b'\x00\x00\x01\x20\x00\x00\x00\x02in6_addr',b'\x00\x01\x34\x11u'),(b'\x00\x00\x01\x32\x00\x00\x00\x10sockaddr',),(b'\x00\x00\x01\x29\x00\x00\x00\x00sockaddr_in6',b'\x00\x00\xCC\x11sin6_family',b'\x00\x00\xCC\x11sin6_port',b'\x00\x00\x16\x11sin6_flowinfo',b'\x00\x01\x20\x11sin6_addr',b'\x00\x01\x36\x11')), - _typenames = (b'\x00\x00\x00\xCCADDRESS_FAMILY',b'\x00\x00\x01\x2FAcceptExPtr',b'\x00\x00\x01\x2EConnectExPtr',b'\x00\x00\x01\x2DDisconnectExPtr',b'\x00\x00\x01\x1EGUID',b'\x00\x00\x01\x20IN6_ADDR',b'\x00\x00\x01\x2DLPFN_DISCONNECTEX',b'\x00\x00\x01\x1FLPIN6_ADDR',b'\x00\x00\x00\x18LPOVERLAPPED',b'\x00\x00\x00\x50LPOVERLAPPED_COMPLETION_ROUTINE',b'\x00\x00\x00\x26LPPROCESS_INFORMATION',b'\x00\x00\x01\x23LPPostCallbackData',b'\x00\x00\x00\xD4LPSECURITY_ATTRIBUTES',b'\x00\x00\x01\x28LPSOCKADDR_IN6_LH',b'\x00\x00\x00\x25LPSTARTUPINFO',b'\x00\x00\x00\x4BLPWSABUF',b'\x00\x00\x01\x21OVERLAPPED',b'\x00\x00\x01\x1FPIN6_ADDR',b'\x00\x00\x01\x22PROCESS_INFORMATION',b'\x00\x00\x01\x25PSCOPE_ID',b'\x00\x00\x00\xD4PSECURITY_ATTRIBUTES',b'\x00\x00\x01\x28PSOCKADDR_IN6_LH',b'\x00\x00\x01\x24PostCallbackData',b'\x00\x00\x01\x26SCOPE_ID',b'\x00\x00\x01\x27SECURITY_ATTRIBUTES',b'\x00\x00\x01\x29SOCKADDR_IN6_LH',b'\x00\x00\x00\x11SOCKET',b'\x00\x00\x01\x2ASTARTUPINFO',b'\x00\x00\x00\x31WAITORTIMERCALLBACK',b'\x00\x00\x01\x2BWSABUF',b'\x00\x00\x00\xCCwint_t'), + _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x5C\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x36\x03\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x1B\x03\x00\x01\x2F\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x40\x03\x00\x00\x22\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x22\x11\x00\x00\x22\x11\x00\x01\x3B\x03\x00\x01\x30\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x03\x00\x00\x2E\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x2E\x11\x00\x00\x11\x11\x00\x01\x26\x03\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x1D\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x1D\x11\x00\x00\x1C\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x1D\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x1C\x11\x00\x00\x1D\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x3C\x03\x00\x00\x0A\x01\x00\x00\x1C\x11\x00\x00\x1C\x11\x00\x00\x1D\x11\x00\x01\x21\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x59\x11\x00\x00\x0A\x01\x00\x00\x1C\x11\x00\x00\x0A\x01\x00\x00\x1D\x11\x00\x00\x5E\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x1C\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x1C\x11\x00\x00\x1C\x03\x00\x00\x1D\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x1C\x11\x00\x00\x1C\x11\x00\x00\x1C\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x1D\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x1C\x11\x00\x00\x1D\x11\x00\x00\x5E\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x1C\x11\x00\x00\x1D\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x2E\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x11\x11\x00\x01\x5C\x03\x00\x00\x0A\x01\x00\x00\x1C\x11\x00\x00\x1D\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\xDD\x03\x00\x00\x07\x01\x00\x01\x3F\x03\x00\x00\x15\x11\x00\x00\x01\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\xAE\x11\x00\x00\xAE\x11\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\xAE\x11\x00\x00\xAE\x11\x00\x00\x2A\x11\x00\x00\x2B\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x6B\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x1B\x0D\x00\x00\x0A\x01\x00\x00\x2E\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x1B\x0D\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x1B\x0D\x00\x00\x11\x11\x00\x00\xAE\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x1B\x0D\x00\x00\x02\x0F\x00\x00\xD8\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\xD8\x0D\x00\x00\x00\x0F\x00\x00\xD8\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x01\x35\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x01\x40\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xE0\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xDD\x03\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xE3\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xE0\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xE3\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xE0\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x11\x11\x00\x00\x11\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xAE\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xE0\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\xE9\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xE0\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x02\x0F\x00\x01\x5C\x0D\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x11\x11\x00\x00\x00\x0F\x00\x01\x5C\x0D\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x0C\x09\x00\x01\x2C\x03\x00\x00\x13\x09\x00\x01\x2E\x03\x00\x00\x14\x09\x00\x00\x0D\x09\x00\x00\x09\x09\x00\x01\x32\x03\x00\x00\x0E\x09\x00\x01\x34\x03\x00\x00\x0A\x09\x00\x00\x0F\x09\x00\x00\x15\x09\x00\x01\x3A\x03\x00\x01\x39\x03\x00\x00\x17\x09\x00\x00\x16\x09\x00\x00\x0B\x09\x00\x00\x10\x09\x00\x01\x3E\x03\x00\x00\x11\x09\x00\x00\x12\x09\x00\x00\x02\x01\x00\x01\x40\x05\x00\x00\x00\x0E\x00\x01\x40\x05\x00\x00\x00\x08\x00\x00\x48\x03\x00\x00\x4E\x03\x00\x00\x93\x03\x00\x00\x05\x01\x00\x00\x01\x09\x00\x00\x04\x09\x00\x00\x07\x09\x00\x00\x08\x09\x00\x00\x00\x09\x00\x00\x02\x09\x00\x00\x03\x09\x00\x00\x05\x09\x00\x00\x06\x09\x00\x01\x53\x03\x00\x00\x04\x01\x00\x01\x53\x05\x00\x00\x00\x10\x00\x01\x53\x05\x00\x00\x00\x08\x00\x00\x1B\x05\x00\x00\x00\x07\x00\x00\xD8\x05\x00\x00\x00\x08\x00\x00\x00\x01\x00\x00\xDD\x05\x00\x00\x01\x00', + _globals = (b'\x00\x00\x3B\x23CancelIo',0,b'\x00\x00\x3E\x23CancelIoEx',0,b'\x00\x00\x3B\x23CloseHandle',0,b'\x00\x00\x3E\x23ConnectNamedPipe',0,b'\x00\x00\xDF\x23CreateEventA',0,b'\x00\x00\xE5\x23CreateEventW',0,b'\x00\x00\xEB\x23CreateFileA',0,b'\x00\x01\x18\x23CreateFileW',0,b'\x00\x01\x06\x23CreateIoCompletionPort',0,b'\x00\x00\xF4\x23CreateNamedPipeA',0,b'\x00\x01\x0E\x23CreateNamedPipeW',0,b'\x00\x00\x2D\x23CreatePipe',0,b'\x00\x00\x21\x23CreateProcessA',0,b'\x00\x00\xB4\x23CreateProcessW',0,b'\x00\x00\x9D\x23DuplicateHandle',0,b'\x00\x01\x0C\x23GetCurrentProcess',0,b'\x00\x00\x6D\x23GetExitCodeProcess',0,b'\x00\x00\xD5\x23GetLastError',0,b'\x00\x00\xD0\x23GetModuleFileNameW',0,b'\x00\x00\x42\x23GetOverlappedResult',0,b'\x00\x00\x71\x23GetQueuedCompletionStatus',0,b'\x00\x01\x03\x23GetStdHandle',0,b'\x00\x00\xD5\x23GetVersion',0,b'\xFF\xFF\xFF\x1FMAX_PROTOCOL_CHAIN',7,b'\x00\x00\x7E\x23PostQueuedCompletionStatus',0,b'\x00\x00\x18\x23ReadFile',0,b'\x00\x00\x33\x23RegisterWaitForSingleObject',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\xC3\x23SetErrorMode',0,b'\x00\x00\x3B\x23SetEvent',0,b'\x00\x00\x78\x23SetNamedPipeHandleState',0,b'\x00\x00\x69\x23TerminateProcess',0,b'\x00\x00\x3B\x23UnregisterWait',0,b'\x00\x00\x8F\x23UnregisterWaitEx',0,b'\x00\x00\x84\x23WSAIoctl',0,b'\xFF\xFF\xFF\x1FWSAPROTOCOL_LEN',255,b'\x00\x00\x57\x23WSARecv',0,b'\x00\x00\x60\x23WSASend',0,b'\x00\x00\xAD\x23WSAStringToAddressW',0,b'\xFF\xFF\xFF\x1FWT_EXECUTEINWAITTHREAD',4,b'\xFF\xFF\xFF\x1FWT_EXECUTEONLYONCE',8,b'\x00\x00\xC6\x23WaitForMultipleObjects',0,b'\x00\x00\xCC\x23WaitForSingleObject',0,b'\x00\x00\xA6\x23WriteFile',0,b'\x00\x00\xC0\x23_get_osfhandle',0,b'\x00\x00\x1F\x23_getch',0,b'\x00\x00\x1F\x23_getche',0,b'\x00\x00\xDA\x23_getwch',0,b'\x00\x00\xDA\x23_getwche',0,b'\x00\x00\x1F\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\xDC\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\xD7\x23_ungetwch',0,b'\x00\x00\x13\x23bind',0,b'\x00\x00\x10\x23closesocket',0,b'\x00\x00\xD7\x23htons',0,b'\x00\x00\xFE\x23socket',0), + _struct_unions = ((b'\x00\x00\x01\x4D\x00\x00\x00\x03$1',b'\x00\x01\x49\x11DUMMYSTRUCTNAME',b'\x00\x00\x11\x11Pointer'),(b'\x00\x00\x01\x49\x00\x00\x00\x02$2',b'\x00\x00\x1B\x11Offset',b'\x00\x00\x1B\x11OffsetHigh'),(b'\x00\x00\x01\x4E\x00\x00\x00\x03$3',b'\x00\x01\x54\x11Byte',b'\x00\x01\x5A\x11Word'),(b'\x00\x00\x01\x4F\x00\x00\x00\x01$4',b'\x00\x01\x4A\x11',b'\x00\x00\x1B\x11Value'),(b'\x00\x00\x01\x4A\x00\x00\x00\x02$5',b'\x00\x00\x1B\x13\x00\x00\x00\x1CZone',b'\x00\x00\x1B\x13\x00\x00\x00\x04Level'),(b'\x00\x00\x01\x50\x00\x00\x00\x03$6',b'\x00\x00\x1B\x11sin6_scope_id',b'\x00\x01\x34\x11sin6_scope_struct'),(b'\x00\x00\x01\x51\x00\x00\x00\x03$7',b'\x00\x01\x4B\x11S_un_b',b'\x00\x01\x4C\x11S_un_w',b'\x00\x00\x1B\x11S_addr'),(b'\x00\x00\x01\x4B\x00\x00\x00\x02$8',b'\x00\x01\x53\x11s_b1',b'\x00\x01\x53\x11s_b2',b'\x00\x01\x53\x11s_b3',b'\x00\x01\x53\x11s_b4'),(b'\x00\x00\x01\x4C\x00\x00\x00\x02$9',b'\x00\x00\xD8\x11s_w1',b'\x00\x00\xD8\x11s_w2'),(b'\x00\x00\x01\x30\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x11\x11hProcess',b'\x00\x00\x11\x11hThread',b'\x00\x00\x1B\x11dwProcessId',b'\x00\x00\x1B\x11dwThreadId'),(b'\x00\x00\x01\x34\x00\x00\x00\x00$SCOPE_ID',b'\x00\x01\x4F\x11'),(b'\x00\x00\x01\x3B\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x1B\x11cb',b'\x00\x00\x22\x11lpReserved',b'\x00\x00\x22\x11lpDesktop',b'\x00\x00\x22\x11lpTitle',b'\x00\x00\x1B\x11dwX',b'\x00\x00\x1B\x11dwY',b'\x00\x00\x1B\x11dwXSize',b'\x00\x00\x1B\x11dwYSize',b'\x00\x00\x1B\x11dwXCountChars',b'\x00\x00\x1B\x11dwYCountChars',b'\x00\x00\x1B\x11dwFillAttribute',b'\x00\x00\x1B\x11dwFlags',b'\x00\x00\xD8\x11wShowWindow',b'\x00\x00\xD8\x11cbReserved2',b'\x00\x01\x52\x11lpReserved2',b'\x00\x00\x11\x11hStdInput',b'\x00\x00\x11\x11hStdOutput',b'\x00\x00\x11\x11hStdError'),(b'\x00\x00\x01\x2A\x00\x00\x00\x02_GUID',b'\x00\x00\x1B\x11Data1',b'\x00\x00\xD8\x11Data2',b'\x00\x00\xD8\x11Data3',b'\x00\x01\x56\x11Data4'),(b'\x00\x00\x01\x2F\x00\x00\x00\x02_OVERLAPPED',b'\x00\x00\x1B\x11Internal',b'\x00\x00\x1B\x11InternalHigh',b'\x00\x01\x4D\x11DUMMYUNIONNAME',b'\x00\x00\x11\x11hEvent'),(b'\x00\x00\x01\x32\x00\x00\x00\x02_PostCallbackData',b'\x00\x00\x11\x11hCompletionPort',b'\x00\x00\x1D\x11Overlapped'),(b'\x00\x00\x01\x35\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x1B\x11nLength',b'\x00\x00\x11\x11lpSecurityDescriptor',b'\x00\x00\x01\x11bInheritHandle'),(b'\x00\x00\x01\x3C\x00\x00\x00\x02_WSABUF',b'\x00\x00\x1B\x11len',b'\x00\x00\x22\x11buf'),(b'\x00\x00\x01\x3E\x00\x00\x00\x02_WSAPROTOCOLCHAIN',b'\x00\x00\x01\x11ChainLen',b'\x00\x01\x58\x11ChainEntries'),(b'\x00\x00\x01\x3F\x00\x00\x00\x02_WSAPROTOCOL_INFOW',b'\x00\x00\x1B\x11dwServiceFlags1',b'\x00\x00\x1B\x11dwServiceFlags2',b'\x00\x00\x1B\x11dwServiceFlags3',b'\x00\x00\x1B\x11dwServiceFlags4',b'\x00\x00\x1B\x11dwProviderFlags',b'\x00\x01\x2A\x11ProviderId',b'\x00\x00\x1B\x11dwCatalogEntryId',b'\x00\x01\x3E\x11ProtocolChain',b'\x00\x00\x01\x11iVersion',b'\x00\x00\x01\x11iAddressFamily',b'\x00\x00\x01\x11iMaxSockAddr',b'\x00\x00\x01\x11iMinSockAddr',b'\x00\x00\x01\x11iSocketType',b'\x00\x00\x01\x11iProtocol',b'\x00\x00\x01\x11iProtocolMaxOffset',b'\x00\x00\x01\x11iNetworkByteOrder',b'\x00\x00\x01\x11iSecurityScheme',b'\x00\x00\x1B\x11dwMessageSize',b'\x00\x00\x1B\x11dwProviderReserved',b'\x00\x01\x5D\x11szProtocol'),(b'\x00\x00\x01\x2C\x00\x00\x00\x02in6_addr',b'\x00\x01\x4E\x11u'),(b'\x00\x00\x01\x2E\x00\x00\x00\x02in_addr',b'\x00\x01\x51\x11S_un'),(b'\x00\x00\x01\x36\x00\x00\x00\x02sockaddr',b'\x00\x00\xD8\x11sa_family',b'\x00\x01\x41\x11sa_data'),(b'\x00\x00\x01\x3A\x00\x00\x00\x02sockaddr_in',b'\x00\x01\x48\x11sin_family',b'\x00\x00\xD8\x11sin_port',b'\x00\x01\x2E\x11sin_addr',b'\x00\x01\x43\x11sin_zero'),(b'\x00\x00\x01\x39\x00\x00\x00\x00sockaddr_in6',b'\x00\x00\xD8\x11sin6_family',b'\x00\x00\xD8\x11sin6_port',b'\x00\x00\x1B\x11sin6_flowinfo',b'\x00\x01\x2C\x11sin6_addr',b'\x00\x01\x50\x11')), + _typenames = (b'\x00\x00\x00\xD8ADDRESS_FAMILY',b'\x00\x00\x01\x47AcceptExPtr',b'\x00\x00\x01\x46ConnectExPtr',b'\x00\x00\x01\x45DisconnectExPtr',b'\x00\x00\x01\x2AGUID',b'\x00\x00\x01\x2CIN6_ADDR',b'\x00\x00\x01\x2EINADDR',b'\x00\x00\x01\x45LPFN_DISCONNECTEX',b'\x00\x00\x01\x2BLPIN6_ADDR',b'\x00\x00\x00\x1DLPOVERLAPPED',b'\x00\x00\x00\x5ELPOVERLAPPED_COMPLETION_ROUTINE',b'\x00\x00\x00\x2BLPPROCESS_INFORMATION',b'\x00\x00\x01\x31LPPostCallbackData',b'\x00\x00\x00\xE0LPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x15LPSOCKADDR',b'\x00\x00\x01\x37LPSOCKADDR_IN',b'\x00\x00\x01\x38LPSOCKADDR_IN6_LH',b'\x00\x00\x00\x2ALPSTARTUPINFO',b'\x00\x00\x00\x59LPWSABUF',b'\x00\x00\x01\x3DLPWSAPROTOCOLCHAIN',b'\x00\x00\x00\xB0LPWSAPROTOCOL_INFOW',b'\x00\x00\x01\x2FOVERLAPPED',b'\x00\x00\x01\x2BPIN6_ADDR',b'\x00\x00\x01\x2DPINADDR',b'\x00\x00\x01\x30PROCESS_INFORMATION',b'\x00\x00\x01\x33PSCOPE_ID',b'\x00\x00\x00\xE0PSECURITY_ATTRIBUTES',b'\x00\x00\x00\x15PSOCKADDR',b'\x00\x00\x01\x37PSOCKADDR_IN',b'\x00\x00\x01\x38PSOCKADDR_IN6_LH',b'\x00\x00\x01\x32PostCallbackData',b'\x00\x00\x01\x34SCOPE_ID',b'\x00\x00\x01\x35SECURITY_ATTRIBUTES',b'\x00\x00\x01\x36SOCKADDR',b'\x00\x00\x01\x3ASOCKADDR_IN',b'\x00\x00\x01\x39SOCKADDR_IN6_LH',b'\x00\x00\x00\x11SOCKET',b'\x00\x00\x01\x3BSTARTUPINFO',b'\x00\x00\x00\x36WAITORTIMERCALLBACK',b'\x00\x00\x01\x3CWSABUF',b'\x00\x00\x01\x3EWSAPROTOCOLCHAIN',b'\x00\x00\x01\x3FWSAPROTOCOL_INFOW',b'\x00\x00\x00\xD8wint_t'), ) From pypy.commits at gmail.com Sun Mar 17 16:25:32 2019 From: pypy.commits at gmail.com (andrewjlawrence) Date: Sun, 17 Mar 2019 13:25:32 -0700 (PDT) Subject: [pypy-commit] pypy winoverlapped: Ensure that WSAStartup is called. Message-ID: <5c8ead3c.1c69fb81.8fecc.547e@mx.google.com> Author: andrewjlawrence Branch: winoverlapped Changeset: r96335:727888c771e9 Date: 2019-03-17 20:24 +0000 http://bitbucket.org/pypy/pypy/changeset/727888c771e9/ Log: Ensure that WSAStartup is called. diff --git a/lib_pypy/_overlapped.py b/lib_pypy/_overlapped.py --- a/lib_pypy/_overlapped.py +++ b/lib_pypy/_overlapped.py @@ -111,6 +111,8 @@ def initiailize_function_ptrs(): + ## importing socket ensures that WSAStartup() is called + import _socket s = _winsock2.socket(AF_INET, SOCK_STREAM, IPPROTO_TCP) dwBytes = _ffi.new("DWORD[1]", [0]) if s == INVALID_SOCKET: From pypy.commits at gmail.com Mon Mar 18 19:15:38 2019 From: pypy.commits at gmail.com (mattip) Date: Mon, 18 Mar 2019 16:15:38 -0700 (PDT) Subject: [pypy-commit] buildbot default: update documentation, unify s390x into regular builders Message-ID: <5c90269a.1c69fb81.c425e.7c2b@mx.google.com> Author: Matti Picus Branch: Changeset: r1078:3049773f1d1a Date: 2019-03-19 01:15 +0200 http://bitbucket.org/pypy/buildbot/changeset/3049773f1d1a/ Log: update documentation, unify s390x into regular builders diff --git a/README-CHROOT b/README-CHROOT --- a/README-CHROOT +++ b/README-CHROOT @@ -50,7 +50,7 @@ chmod buildslave.buildslave -R /extra1/stretch32/home/buildslave apt install virtualenv buildbot-slave python-pytest python-hypothesis \ -netbase gdb ncurses-term pypy locales +netbase gdb ncurses-term pypy locales mercurial and more from the instructions on https://bitbucket.org/pypy/buildbot diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -292,8 +292,10 @@ LINUX32OWN, # on benchmarker4_32, uses all cores LINUX64OWN, # on bencher4, uses all cores WIN32OWN, # on SalsaSalsa + LINUX_S390XOWN, JITLINUX32, # on benchmarker4_32, uses 1 core JITLINUX64, # on bencher4, uses 1 core + JITLINUX_S390X, #APPLVLLINUX32, #APPLVLLINUX64, # on bencher4, uses 1 core # other platforms @@ -350,14 +352,6 @@ #onlyIfChanged=True, ), - # S390X vm (ibm-research) - Nightly("nightly-4-00", [ - LINUX_S390XOWN, - ], branch='default', hour=0, minute=0), - Nightly("nightly-4-01", [JITLINUX_S390X], branch='default', hour=2, minute=0, - onlyIfChanged=True, - ), - # this one has faithfully run every night even though the latest # change to that branch was in January 2013. Re-enable one day. #Nightly("nighly-ppc", [ From pypy.commits at gmail.com Tue Mar 19 16:19:42 2019 From: pypy.commits at gmail.com (mattip) Date: Tue, 19 Mar 2019 13:19:42 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: add option for non-x86 (s390x) Message-ID: <5c914ede.1c69fb81.11a26.1ad8@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r96336:506b37a575b5 Date: 2019-03-19 22:18 +0200 http://bitbucket.org/pypy/pypy/changeset/506b37a575b5/ Log: add option for non-x86 (s390x) diff --git a/lib_pypy/_blake2/_blake2_build.py b/lib_pypy/_blake2/_blake2_build.py --- a/lib_pypy/_blake2/_blake2_build.py +++ b/lib_pypy/_blake2/_blake2_build.py @@ -13,8 +13,11 @@ elif IS_WIN: extra_compile_args = [] define_macros = [('__SSE2__', '1')] +elif platform.machine().startswith('x86'): + extra_compile_args = ['-msse2'] + define_macros = [] else: - extra_compile_args = ['-msse2'] + extra_compile_args = [] define_macros = [] From pypy.commits at gmail.com Wed Mar 20 04:28:15 2019 From: pypy.commits at gmail.com (arigo) Date: Wed, 20 Mar 2019 01:28:15 -0700 (PDT) Subject: [pypy-commit] cffi default: Issue #406: document the failure Message-ID: <5c91f99f.1c69fb81.7fb8c.4430@mx.google.com> Author: Armin Rigo Branch: Changeset: r3247:000711129df0 Date: 2019-03-20 09:27 +0100 http://bitbucket.org/cffi/cffi/changeset/000711129df0/ Log: Issue #406: document the failure diff --git a/testing/cffi0/test_verify.py b/testing/cffi0/test_verify.py --- a/testing/cffi0/test_verify.py +++ b/testing/cffi0/test_verify.py @@ -2099,6 +2099,11 @@ raise errors[0][1] def test_errno_working_even_with_pypys_jit(): + # NOTE: on some platforms, to work correctly, this test needs to be + # compiled with -pthread. Otherwise, the accesses to errno done from f() + # are compiled by assuming this small library won't be used from multiple + # threads, which is wrong. If you see failures _and_ if you pass your + # own CFLAGS environment variable, please make sure "-pthread" is in it. ffi = FFI() ffi.cdef("int f(int);") lib = ffi.verify(""" From pypy.commits at gmail.com Wed Mar 20 04:59:50 2019 From: pypy.commits at gmail.com (mattip) Date: Wed, 20 Mar 2019 01:59:50 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: mimic more closely what cpython does in Modules/_blake2/_blake2?_impl.c Message-ID: <5c920106.1c69fb81.93c14.7247@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r96337:0da5abfd7fd5 Date: 2019-03-20 10:59 +0200 http://bitbucket.org/pypy/pypy/changeset/0da5abfd7fd5/ Log: mimic more closely what cpython does in Modules/_blake2/_blake2?_impl.c diff --git a/lib_pypy/_blake2/_blake2_build.py b/lib_pypy/_blake2/_blake2_build.py --- a/lib_pypy/_blake2/_blake2_build.py +++ b/lib_pypy/_blake2/_blake2_build.py @@ -4,19 +4,17 @@ from cffi import FFI -IS_ARM = platform.machine().startswith('arm') IS_WIN = sys.platform == 'win32' -if IS_ARM: - # XXX Choose neon accelaration - define_macros = [] - extra_compile_args = [] -elif IS_WIN: +if IS_WIN: + BLAKE2_USE_SSE = True extra_compile_args = [] define_macros = [('__SSE2__', '1')] elif platform.machine().startswith('x86'): + BLAKE2_USE_SSE = True extra_compile_args = ['-msse2'] define_macros = [] else: + BLAKE2_USE_SSE = False extra_compile_args = [] define_macros = [] @@ -83,13 +81,18 @@ _libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'impl')) +if BLAKE2_USE_SSE: + sourcesB=[os.path.join(_libdir, 'blake2b.c'), ], + sourcesS=[os.path.join(_libdir, 'blake2s.c'), ], +else: + sourcesB=[os.path.join(_libdir, 'blake2b-ref.c'), ], + sourcesS=[os.path.join(_libdir, 'blake2s-ref.c'), ], blake2b_ffi = FFI() blake2b_ffi.cdef(blake_cdef) blake2b_ffi.set_source( '_blake2b_cffi', blake2b_source, - sources=[os.path.join(_libdir, 'blake2b.c'), - ], + sources=sourcesB, include_dirs=[_libdir], extra_compile_args=extra_compile_args, define_macros=define_macros, @@ -105,8 +108,7 @@ blake2s_ffi.cdef(blake_cdef) blake2s_ffi.set_source( '_blake2s_cffi', _replace_b2s(blake2b_source), - sources=[os.path.join(_libdir, 'blake2s.c'), - ], + sources=sourcesS, include_dirs=[_libdir], extra_compile_args=extra_compile_args, define_macros=define_macros, From pypy.commits at gmail.com Wed Mar 20 12:05:52 2019 From: pypy.commits at gmail.com (mattip) Date: Wed, 20 Mar 2019 09:05:52 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: typo Message-ID: <5c9264e0.1c69fb81.3d436.7ad6@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r96338:2583857fdf26 Date: 2019-03-20 18:05 +0200 http://bitbucket.org/pypy/pypy/changeset/2583857fdf26/ Log: typo diff --git a/lib_pypy/_blake2/_blake2_build.py b/lib_pypy/_blake2/_blake2_build.py --- a/lib_pypy/_blake2/_blake2_build.py +++ b/lib_pypy/_blake2/_blake2_build.py @@ -82,11 +82,11 @@ _libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'impl')) if BLAKE2_USE_SSE: - sourcesB=[os.path.join(_libdir, 'blake2b.c'), ], - sourcesS=[os.path.join(_libdir, 'blake2s.c'), ], + sourcesB=[os.path.join(_libdir, 'blake2b.c'), ] + sourcesS=[os.path.join(_libdir, 'blake2s.c'), ] else: - sourcesB=[os.path.join(_libdir, 'blake2b-ref.c'), ], - sourcesS=[os.path.join(_libdir, 'blake2s-ref.c'), ], + sourcesB=[os.path.join(_libdir, 'blake2b-ref.c'), ] + sourcesS=[os.path.join(_libdir, 'blake2s-ref.c'), ] blake2b_ffi = FFI() blake2b_ffi.cdef(blake_cdef) From pypy.commits at gmail.com Thu Mar 21 08:14:32 2019 From: pypy.commits at gmail.com (mattip) Date: Thu, 21 Mar 2019 05:14:32 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy3.6-v7.x: merge py3.6 into release Message-ID: <5c938028.1c69fb81.5a4e8.9b7d@mx.google.com> Author: Matti Picus Branch: release-pypy3.6-v7.x Changeset: r96339:7a2e437acfce Date: 2019-03-21 14:13 +0200 http://bitbucket.org/pypy/pypy/changeset/7a2e437acfce/ Log: merge py3.6 into release diff --git a/lib_pypy/_blake2/_blake2_build.py b/lib_pypy/_blake2/_blake2_build.py --- a/lib_pypy/_blake2/_blake2_build.py +++ b/lib_pypy/_blake2/_blake2_build.py @@ -4,17 +4,18 @@ from cffi import FFI -IS_ARM = platform.machine().startswith('arm') IS_WIN = sys.platform == 'win32' -if IS_ARM: - # XXX Choose neon accelaration - define_macros = [] - extra_compile_args = [] -elif IS_WIN: +if IS_WIN: + BLAKE2_USE_SSE = True extra_compile_args = [] define_macros = [('__SSE2__', '1')] +elif platform.machine().startswith('x86'): + BLAKE2_USE_SSE = True + extra_compile_args = ['-msse2'] + define_macros = [] else: - extra_compile_args = ['-msse2'] + BLAKE2_USE_SSE = False + extra_compile_args = [] define_macros = [] @@ -80,13 +81,18 @@ _libdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'impl')) +if BLAKE2_USE_SSE: + sourcesB=[os.path.join(_libdir, 'blake2b.c'), ] + sourcesS=[os.path.join(_libdir, 'blake2s.c'), ] +else: + sourcesB=[os.path.join(_libdir, 'blake2b-ref.c'), ] + sourcesS=[os.path.join(_libdir, 'blake2s-ref.c'), ] blake2b_ffi = FFI() blake2b_ffi.cdef(blake_cdef) blake2b_ffi.set_source( '_blake2b_cffi', blake2b_source, - sources=[os.path.join(_libdir, 'blake2b.c'), - ], + sources=sourcesB, include_dirs=[_libdir], extra_compile_args=extra_compile_args, define_macros=define_macros, @@ -102,8 +108,7 @@ blake2s_ffi.cdef(blake_cdef) blake2s_ffi.set_source( '_blake2s_cffi', _replace_b2s(blake2b_source), - sources=[os.path.join(_libdir, 'blake2s.c'), - ], + sources=sourcesS, include_dirs=[_libdir], extra_compile_args=extra_compile_args, define_macros=define_macros, diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py --- a/pypy/tool/build_cffi_imports.py +++ b/pypy/tool/build_cffi_imports.py @@ -224,7 +224,7 @@ print("stdout:") print(stdout.decode('utf-8')) print("stderr:") - print(stderr.decode('utf-8')) + print(stderr) except: import traceback;traceback.print_exc() failures.append((key, module)) From pypy.commits at gmail.com Thu Mar 21 08:14:35 2019 From: pypy.commits at gmail.com (mattip) Date: Thu, 21 Mar 2019 05:14:35 -0700 (PDT) Subject: [pypy-commit] pypy default: Moved tag release-pypy3.6-v7.1.0 to changeset 7a2e437acfce (from changeset 6fd188f8f903) Message-ID: <5c93802b.1c69fb81.5ca4.f3f4@mx.google.com> Author: Matti Picus Branch: Changeset: r96340:b2c0cc585997 Date: 2019-03-21 14:13 +0200 http://bitbucket.org/pypy/pypy/changeset/b2c0cc585997/ Log: Moved tag release-pypy3.6-v7.1.0 to changeset 7a2e437acfce (from changeset 6fd188f8f903) diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -71,3 +71,5 @@ bb0d05b190b9c579f0c889a368636e14f6205bab release-pypy3.6-v7.1.0 bb0d05b190b9c579f0c889a368636e14f6205bab release-pypy3.6-v7.1.0 6fd188f8f903b7555920adf7d5e7fe21db1bd593 release-pypy3.6-v7.1.0 +6fd188f8f903b7555920adf7d5e7fe21db1bd593 release-pypy3.6-v7.1.0 +7a2e437acfceafe2665b23b1394dc6c66add3b89 release-pypy3.6-v7.1.0 From pypy.commits at gmail.com Fri Mar 22 03:49:36 2019 From: pypy.commits at gmail.com (mattip) Date: Fri, 22 Mar 2019 00:49:36 -0700 (PDT) Subject: [pypy-commit] pypy.org extradoc: update sha256 for new uploads, add s390x for py3.6 Message-ID: <5c949390.1c69fb81.a1b63.b4b3@mx.google.com> Author: Matti Picus Branch: extradoc Changeset: r939:cd3100982679 Date: 2019-03-22 09:49 +0200 http://bitbucket.org/pypy/pypy.org/changeset/cd3100982679/ Log: update sha256 for new uploads, add s390x for py3.6 diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -162,6 +162,7 @@ * `Linux x86 binary (32bit, built on Ubuntu 12.04 - 16.04)`__ (see ``[1]`` below) * `Mac OS X binary (64bit)`__ (High Sierra >= 10.13, not for Sierra and below) * `Windows binary (32bit)`__ +* `s390x Linux binary (built on Redhat Linux 7.2)`__ (see ``[1]`` below) * `Source (tar.bz2)`__; `Source (zip)`__. See below for more about the sources. * `All our downloads,`__ including previous versions. We also have a mirror_, but please use only if you have troubles accessing the links above @@ -170,6 +171,7 @@ .. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.0-linux32.tar.bz2 .. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.0-osx64.tar.bz2 .. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.0-win32.zip +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.0-s390x.tar.bz2 .. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.0-src.tar.bz2 .. __: https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.1.0-src.zip .. __: https://bitbucket.org/pypy/pypy/downloads @@ -461,22 +463,23 @@ pypy2.7-7.1.0 sha256:: - deb13950ead91668db64e3a375e1e3c5ff041fe5a6e12d16887ef98b7946b046 pypy2.7-v7.1.0-linux32.tar.bz2 - a3e467eef5e048484ac35faa46dec99818689735926c69376b3f0a59637b1593 pypy2.7-v7.1.0-linux64.tar.bz2 - 6a40d230730ca2dc898ceba1f4022130f36bb599235cde976adc51086eaf2378 pypy2.7-v7.1.0-osx64.tar.bz2 - 82ee6158c5485df6d50812095bb82c7490519d8160d07f3dec4dc21933c473f5 pypy2.7-v7.1.0-s390x.tar.bz2 - 84ce5bc2867b224e2516ef431d78c6908d0182bc89444f7c1ef707443763754f pypy2.7-v7.1.0-src.tar.bz2 - 609a3d07e66f78a8001215abdfb3bacbc2f7cfeda141ccf1cac45c5249678521 pypy2.7-v7.1.0-src.zip - bb10276ff100c0208b11d6dfe95c53ea66fc90a1356a1094cb50ed1394b95eb7 pypy2.7-v7.1.0-win32.zip + 44ec91e8cb01caab289d8763c203f3aaf288d14325a6c42692bd1ac4e870d758 pypy2.7-v7.1.0-linux32.tar.bz2 + fef176a29a2ef068c00c8098e59dab935ca6e956f089672b3f7351da95a034f5 pypy2.7-v7.1.0-linux64.tar.bz2 + 8be43685ce718b0768387450fc6dc395d60809b778b6146c353ef67826022153 pypy2.7-v7.1.0-osx64.tar.bz2 + b065f55741bcb37863f1eca30ce91c9d79159371a6994100930cdc2ede3237bc pypy2.7-v7.1.0-s390x.tar.bz2 + b051a71ea5b4fa27d0a744b28e6054661adfce8904dcc82500716b5edff5ce4b pypy2.7-v7.1.0-src.tar.bz2 + e60ce30f9947844da43daaa7658adc0c05330681305225954114772f42df06ec pypy2.7-v7.1.0-src.zip + 76658c9ad679d562b8b6a09d006caa666406337b9834ff56db16980c5e549f20 pypy2.7-v7.1.0-win32.zip pypy 3.6-v7.1.0 sha256:: - 315432517b09a5ec4afa91517f0af9e37c73557a4751a111ee88230a36061399 pypy3.6-v7.1.0-linux32.tar.bz2 - 580a3cc01517783aa24f4d5efe88b4323106d6952f9cf987adb11c909f11aa54 pypy3.6-v7.1.0-linux64.tar.bz2 - d68229c1755fa2d22fb1449be48a6f8681c8def0a3c93dae2929b95a0d535ff5 pypy3.6-v7.1.0-osx64.tar.bz2 - 181cd333b5aafa4bf041a1b249113ab493fb63d1880504e5912109d9aab74689 pypy3.6-v7.1.0-src.tar.bz2 - d2ba6f2d5a0e5a20edf525488912b86a92f296347481ffaa28b7a293ad48a4cd pypy3.6-v7.1.0-src.zip - 2b55b7e03e415790e19a45b8b63d86be20c9d4e1297b84e624d6bc275680d0fa pypy3.6-v7.1.0-win32.zip + 6e1fd6a820c8ba850761b052a1e53c2138d2a46d221202dd010815d7c9723269 pypy3.6-v7.1.0-linux32.tar.bz2 + 853ab874ae3b1f929576890c511f3efbc9159c2a16332113e68be7dccc51e2aa pypy3.6-v7.1.0-linux64.tar.bz2 + 84f8fb41aa7ec6e3f72546d1ea1d412f7ef51edf039074610295c59cc0ff1e35 pypy3.6-v7.1.0-osx64.tar.bz2 + 9ebc7bee180432991dd8a2f15d320120cf99ef1a315e764e9ba6ff4b969819b8 pypy3.6-v7.1.0-s390x.tar.bz2 + 3e3091798cff9e7b6de35f7aa7f71965e06dcbf35112eba202d4ded7675f95f4 pypy3.6-v7.1.0-src.tar.bz2 + e156b8cf13feae37bc4a8ea95d137c0422255188137279085917e65ec0b5731a pypy3.6-v7.1.0-src.zip + 1383497f89e034e4124ea3e65979b0701b0296c06f4fec7082c8a39014a821f0 pypy3.6-v7.1.0-win32.zip pypy2.7-7.0.0 sha256:: From pypy.commits at gmail.com Fri Mar 22 06:08:47 2019 From: pypy.commits at gmail.com (mattip) Date: Fri, 22 Mar 2019 03:08:47 -0700 (PDT) Subject: [pypy-commit] pypy default: document proposal to tag release candidates, for next time Message-ID: <5c94b42f.1c69fb81.4246f.e946@mx.google.com> Author: Matti Picus Branch: Changeset: r96341:b77fbcee4e98 Date: 2019-03-22 12:07 +0200 http://bitbucket.org/pypy/pypy/changeset/b77fbcee4e98/ Log: document proposal to tag release candidates, for next time diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst --- a/pypy/doc/how-to-release.rst +++ b/pypy/doc/how-to-release.rst @@ -40,11 +40,11 @@ $ hg up -r default $ # edit the version to e.g. 7.0.0-final $ hg ci - $ hg branch release-pypy2.7-7.x && hg ci + $ hg branch release-pypy2.7-v7.x && hg ci $ hg up -r default $ # edit the version to 7.1.0-alpha0 $ hg ci - $ hg up -r release-pypy2.7-7.x + $ hg up -r release-pypy2.7-v7.x $ hg merge default $ # edit the version to AGAIN 7.0.0-final $ hg ci @@ -53,11 +53,11 @@ $ hg up -r py3.5 $ hg merge default # this brings the version fo 7.1.0-alpha0 - $ hg branch release-pypy3.5-7.x + $ hg branch release-pypy3.5-v7.x $ # edit the version to 7.0.0-final $ hg ci $ hg up -r py3.5 - $ hg merge release-pypy3.5-7.x + $ hg merge release-pypy3.5-v7.x $ # edit the version to 7.1.0-alpha0 $ hg ci @@ -109,9 +109,11 @@ * add a tag on the pypy/jitviewer repo that corresponds to pypy release, so that the source tarball can be produced in the next steps - * download the builds, repackage binaries. Tag the release version - and download and repackage source from bitbucket. You may find it - convenient to use the ``repackage.sh`` script in pypy/tool/release to do this. + * download the builds, repackage binaries. Tag the release-candidate version + (it is important to mark this as a candidate since usually at least two + tries are needed to complete the process) and download and repackage source + from bitbucket. You may find it convenient to use the ``repackage.sh`` + script in pypy/tool/release to do this. Otherwise repackage and upload source "-src.tar.bz2" to bitbucket and to cobra, as some packagers prefer a clearly labeled source package @@ -135,3 +137,5 @@ * add a tag on the codespeed web site that corresponds to pypy release * revise versioning at https://readthedocs.org/projects/pypy + * tag the final release(s) with appropriate tags + diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh --- a/pypy/tool/release/repackage.sh +++ b/pypy/tool/release/repackage.sh @@ -5,7 +5,7 @@ min=1 rev=0 branchname=release-pypy$pmaj.$pmin-v$maj.x # ==OR== release-v$maj.x # ==OR== release-v$maj.$min.x -tagname=release-pypy$pmaj.$pmin-v$maj.$min.$rev # ==OR== release-$maj.$min +tagname=release-canditate-pypy$pmaj.$pmin-v$maj.$min.$rev # ==OR== release-$maj.$min echo checking hg log -r $branchname hg log -r $branchname || exit 1 From pypy.commits at gmail.com Sun Mar 24 12:59:42 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 24 Mar 2019 09:59:42 -0700 (PDT) Subject: [pypy-commit] pypy.org extradoc: update and regenerate Message-ID: <5c97b77e.1c69fb81.637aa.6834@mx.google.com> Author: Matti Picus Branch: extradoc Changeset: r940:a27c0442e7c6 Date: 2019-03-24 18:59 +0200 http://bitbucket.org/pypy/pypy.org/changeset/a27c0442e7c6/ Log: update and regenerate diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -66,11 +66,11 @@ as stable as the release, but they contain numerous bugfixes and performance improvements.

We provide binaries for x86, ARM, PPC and s390x running on different operating systems such as -Linux, Mac OS X and Windows (what's new in PyPy 7.0?):

+Linux, Mac OS X and Windows (what's new in PyPy 7.1?):

    -
  • the Python2.7 compatible release — PyPy2.7 v7.0
    • +
  • the Python2.7 compatible release — PyPy2.7 v7.1
  • the Python3.5 compatible release — PyPy3.5 v7.0
    • -
  • the Python3.6 compatible release, alpha quality — PyPy3.6 v7.0
    • +
  • the Python3.6 compatible release, beta quality — PyPy3.6 v7.1
  • the Python2.7 Software Transactional Memory special release — PyPy-STM 2.5.1 (Linux x86-64 only)
    @@ -109,19 +109,19 @@
  • or translate your own PyPy.
-
-

Python2.7 compatible PyPy 7.0

+
+

Python2.7 compatible PyPy 7.1

@@ -151,13 +151,15 @@ libraries: …. Unless you want to hack a lot, try out the portable Linux binaries.

-
-

Python 3.6 compatible PyPy3.6 v7.0.0-alpha

+
+

Python 3.6 compatible PyPy3.6 v7.1.0-beta

@@ -218,8 +220,6 @@

1. Standard NumPy

Installation works on any recent PyPy (the release above is fine). -The currently released numpy 1.13 works except for nditers with the -updateifcopy flag. For example, without using a virtualenv:

 $ ./pypy-xxx/bin/pypy -m ensurepip
@@ -391,7 +391,27 @@

Checksums

-

Here are the checksums for each of the downloads of PyPy 7.0.0 and the older 6.0.0.

+

Here are the checksums for each of the downloads of PyPy 7.1.0, 7.0.0 and 6.0.0.

+

pypy2.7-7.1.0 sha256:

+
+44ec91e8cb01caab289d8763c203f3aaf288d14325a6c42692bd1ac4e870d758  pypy2.7-v7.1.0-linux32.tar.bz2
+fef176a29a2ef068c00c8098e59dab935ca6e956f089672b3f7351da95a034f5  pypy2.7-v7.1.0-linux64.tar.bz2
+8be43685ce718b0768387450fc6dc395d60809b778b6146c353ef67826022153  pypy2.7-v7.1.0-osx64.tar.bz2
+b065f55741bcb37863f1eca30ce91c9d79159371a6994100930cdc2ede3237bc  pypy2.7-v7.1.0-s390x.tar.bz2
+b051a71ea5b4fa27d0a744b28e6054661adfce8904dcc82500716b5edff5ce4b  pypy2.7-v7.1.0-src.tar.bz2
+e60ce30f9947844da43daaa7658adc0c05330681305225954114772f42df06ec  pypy2.7-v7.1.0-src.zip
+76658c9ad679d562b8b6a09d006caa666406337b9834ff56db16980c5e549f20  pypy2.7-v7.1.0-win32.zip
+
+

pypy 3.6-v7.1.0 sha256:

+
+6e1fd6a820c8ba850761b052a1e53c2138d2a46d221202dd010815d7c9723269  pypy3.6-v7.1.0-linux32.tar.bz2
+853ab874ae3b1f929576890c511f3efbc9159c2a16332113e68be7dccc51e2aa  pypy3.6-v7.1.0-linux64.tar.bz2
+84f8fb41aa7ec6e3f72546d1ea1d412f7ef51edf039074610295c59cc0ff1e35  pypy3.6-v7.1.0-osx64.tar.bz2
+9ebc7bee180432991dd8a2f15d320120cf99ef1a315e764e9ba6ff4b969819b8  pypy3.6-v7.1.0-s390x.tar.bz2
+3e3091798cff9e7b6de35f7aa7f71965e06dcbf35112eba202d4ded7675f95f4  pypy3.6-v7.1.0-src.tar.bz2
+e156b8cf13feae37bc4a8ea95d137c0422255188137279085917e65ec0b5731a  pypy3.6-v7.1.0-src.zip
+1383497f89e034e4124ea3e65979b0701b0296c06f4fec7082c8a39014a821f0  pypy3.6-v7.1.0-win32.zip
+

pypy2.7-7.0.0 sha256:

 446fc208dd77a0048368da830564e6e4180bcd786e524b5369c61785af5c903a  pypy2.7-v7.0.0-linux32.tar.bz2
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -19,7 +19,7 @@
 
 * the Python3.5 compatible release — **PyPy3.5 v7.0**
 
-* the Python3.6 compatible release, alpha quality — **PyPy3.6 v7.1**
+* the Python3.6 compatible release, beta quality — **PyPy3.6 v7.1**
 
 * the Python2.7 Software Transactional Memory special release — **PyPy-STM 2.5.1** (Linux x86-64 only)
 

From pypy.commits at gmail.com  Sun Mar 24 13:50:05 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 24 Mar 2019 10:50:05 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: move paragraph up,
 tighten build description
Message-ID: <5c97c34d.1c69fb81.144b8.dbc3@mx.google.com>

Author: Matti Picus 
Branch: extradoc
Changeset: r941:1923d2950a04
Date: 2019-03-24 19:49 +0200
http://bitbucket.org/pypy/pypy.org/changeset/1923d2950a04/

Log:	move paragraph up, tighten build description

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -112,8 +112,8 @@
 

 
Python2.7 compatible PyPy 7.1

pypy 3.6-v7.1.0 sha256:

-6e1fd6a820c8ba850761b052a1e53c2138d2a46d221202dd010815d7c9723269  pypy3.6-v7.1.0-linux32.tar.bz2
-853ab874ae3b1f929576890c511f3efbc9159c2a16332113e68be7dccc51e2aa  pypy3.6-v7.1.0-linux64.tar.bz2
-84f8fb41aa7ec6e3f72546d1ea1d412f7ef51edf039074610295c59cc0ff1e35  pypy3.6-v7.1.0-osx64.tar.bz2
-9ebc7bee180432991dd8a2f15d320120cf99ef1a315e764e9ba6ff4b969819b8  pypy3.6-v7.1.0-s390x.tar.bz2
-3e3091798cff9e7b6de35f7aa7f71965e06dcbf35112eba202d4ded7675f95f4  pypy3.6-v7.1.0-src.tar.bz2
-e156b8cf13feae37bc4a8ea95d137c0422255188137279085917e65ec0b5731a  pypy3.6-v7.1.0-src.zip
+031bfac61210a6e161bace0691b854dc15d01b0e624dc0588c544ee5e1621a83  pypy3.6-v7.1.0-linux32.tar.bz2
+270dd06633cf03337e6f815d7235e790e90dabba6f4b6345c9745121006925fc  pypy3.6-v7.1.0-linux64.tar.bz2
+d46e005ba095cb4a7006079ffbf4fe63c18cf5e9d8ce9ce8383efc1a4863ab5b  pypy3.6-v7.1.0-osx64.tar.bz2
+243cd0cc188a94c1f064f402ae72b8ba4303eb3137eac53c53826472b8005098  pypy3.6-v7.1.0-s390x.tar.bz2
+faa81f469bb2a7cbd22c64f22d4b4ddc5a1f7c798d43b7919b629b932f9b1c6f  pypy3.6-v7.1.0-src.tar.bz2
+4858e7e8a0007bc3b381bd392208b28d30889a4e5a88a3c28e3d9dc4f25b654e  pypy3.6-v7.1.0-src.zip
 1383497f89e034e4124ea3e65979b0701b0296c06f4fec7082c8a39014a821f0  pypy3.6-v7.1.0-win32.zip

pypy2.7-7.0.0 sha256:

diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -471,12 +471,13 @@ pypy 3.6-v7.1.0 sha256:: - 6e1fd6a820c8ba850761b052a1e53c2138d2a46d221202dd010815d7c9723269 pypy3.6-v7.1.0-linux32.tar.bz2 - 853ab874ae3b1f929576890c511f3efbc9159c2a16332113e68be7dccc51e2aa pypy3.6-v7.1.0-linux64.tar.bz2 - 84f8fb41aa7ec6e3f72546d1ea1d412f7ef51edf039074610295c59cc0ff1e35 pypy3.6-v7.1.0-osx64.tar.bz2 - 9ebc7bee180432991dd8a2f15d320120cf99ef1a315e764e9ba6ff4b969819b8 pypy3.6-v7.1.0-s390x.tar.bz2 - 3e3091798cff9e7b6de35f7aa7f71965e06dcbf35112eba202d4ded7675f95f4 pypy3.6-v7.1.0-src.tar.bz2 - e156b8cf13feae37bc4a8ea95d137c0422255188137279085917e65ec0b5731a pypy3.6-v7.1.0-src.zip + + 031bfac61210a6e161bace0691b854dc15d01b0e624dc0588c544ee5e1621a83 pypy3.6-v7.1.0-linux32.tar.bz2 + 270dd06633cf03337e6f815d7235e790e90dabba6f4b6345c9745121006925fc pypy3.6-v7.1.0-linux64.tar.bz2 + d46e005ba095cb4a7006079ffbf4fe63c18cf5e9d8ce9ce8383efc1a4863ab5b pypy3.6-v7.1.0-osx64.tar.bz2 + 243cd0cc188a94c1f064f402ae72b8ba4303eb3137eac53c53826472b8005098 pypy3.6-v7.1.0-s390x.tar.bz2 + faa81f469bb2a7cbd22c64f22d4b4ddc5a1f7c798d43b7919b629b932f9b1c6f pypy3.6-v7.1.0-src.tar.bz2 + 4858e7e8a0007bc3b381bd392208b28d30889a4e5a88a3c28e3d9dc4f25b654e pypy3.6-v7.1.0-src.zip 1383497f89e034e4124ea3e65979b0701b0296c06f4fec7082c8a39014a821f0 pypy3.6-v7.1.0-win32.zip pypy2.7-7.0.0 sha256:: From pypy.commits at gmail.com Mon Mar 25 02:43:41 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 24 Mar 2019 23:43:41 -0700 (PDT) Subject: [pypy-commit] pypy default: simplify by requiring user to specify exe name Message-ID: <5c98789d.1c69fb81.e11b0.68bc@mx.google.com> Author: Matti Picus Branch: Changeset: r96348:7dfde239f1b9 Date: 2019-03-25 08:43 +0200 http://bitbucket.org/pypy/pypy/changeset/7dfde239f1b9/ Log: simplify by requiring user to specify exe name diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh --- a/pypy/tool/release/repackage.sh +++ b/pypy/tool/release/repackage.sh @@ -1,9 +1,12 @@ # Edit these appropriately before running this script -pmaj=2 # python main version +pmaj=2 # python main version: 2 or 3 pmin=7 # python minor version +exe=pypy3 # pypy3 or pypy maj=7 min=1 rev=0 + + branchname=release-pypy$pmaj.$pmin-v$maj.x # ==OR== release-v$maj.x # ==OR== release-v$maj.$min.x tagname=release-candidate-pypy$pmaj.$pmin-v$maj.$min.$rev # ==OR== release-$maj.$min # tagname=release-pypy$pmaj.$pmin-v$maj.$min.$rev # ==OR== release-$maj.$min @@ -51,15 +54,9 @@ plat_final=linux32 fi mv pypy-c-jit-*-$plat $rel-$plat_final + # TODO: automate the platform choice or move it to the head of the file if [ $plat_final == linux64 ] then - if [ $pmaj == 3 ] - then - exe=pypy3 - else - exe=pypy - fi - # TODO: programatically figure out which platform to use actual_ver=`$rel-$plat_final/bin/$exe -c "import sys; print('.'.join([str(x) for x in sys.pypy_version_info[:2]]))"` fi echo packaging $plat_final From pypy.commits at gmail.com Mon Mar 25 15:07:29 2019 From: pypy.commits at gmail.com (mattip) Date: Mon, 25 Mar 2019 12:07:29 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: override system Exxx that were added in MSVC 2010, cpython decided to stick with the old values Message-ID: <5c9926f1.1c69fb81.9775.604b@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r96349:c9bd74b7f75e Date: 2019-03-25 21:05 +0200 http://bitbucket.org/pypy/pypy/changeset/c9bd74b7f75e/ Log: override system Exxx that were added in MSVC 2010, cpython decided to stick with the old values diff --git a/pypy/module/errno/interp_errno.py b/pypy/module/errno/interp_errno.py --- a/pypy/module/errno/interp_errno.py +++ b/pypy/module/errno/interp_errno.py @@ -43,6 +43,18 @@ "WSAGETASYNCBUFLE", "WSAEDESTADDRREQ", "WSAECONNREFUSED", "WSAENETRESET", "WSAN", "WSAEDQUOT"] +# The following constants were added to errno.h in VS2010 but have +# preferred WSA equivalents, so errno.EADDRINUSE == errno.WSAEADDRINUSE. +win_errors_override = [ + "WSAEADDRINUSE", "WSAEADDRNOTAVAI", "WSAEAFNOSUPPORT", "WSAEALREADY", + "WSAECONNABORTED", "WSAECONNREFUSED", "WSAECONNRESET", "WSAEDESTADDRREQ", + "WSAEHOSTUNREACH", "WSAEINPROGRESS", "WSAEISCONN", "WSAELOOP", + "WSAEMSGSIZE", "WSAENETDOWN", "WSAENETRESET", "WSAENETUNREACH", + "WSAENOBUFS", "WSAENOPROTOOPT", "WSAENOTCONN", "WSAENOTSOCK", + "WSAEOPNOTSUPP", "WSAEPROTONOSUPPORT", "WSAEPROTOTYPE", "WSAETIMEDOUT", + "WSAEWOULDBLOCK", + ] + more_errors = [ "ENOMEDIUM", "EMEDIUMTYPE", "ECANCELED", "ENOKEY", "EKEYEXPIRED", "EKEYREVOKED", "EKEYREJECTED", "EOWNERDEAD", "ENOTRECOVERABLE", "ERFKILL", @@ -80,7 +92,8 @@ assert name.startswith('WSA') code = config[name] if code is not None: - if name[3:] in errors and name[3:] not in name2code: + if name[3:] in errors and (name in win_errors_override or + name[3:] not in name2code): # errno.EFOO = name2code[name[3:]] = code # errno.WSABAR = From pypy.commits at gmail.com Mon Mar 25 16:41:17 2019 From: pypy.commits at gmail.com (mattip) Date: Mon, 25 Mar 2019 13:41:17 -0700 (PDT) Subject: [pypy-commit] pypy.org extradoc: update sha256 hashes for new win32 download Message-ID: <5c993ced.1c69fb81.79fbf.d5be@mx.google.com> Author: Matti Picus Branch: extradoc Changeset: r944:c1b2bd8d0e7a Date: 2019-03-25 22:40 +0200 http://bitbucket.org/pypy/pypy.org/changeset/c1b2bd8d0e7a/ Log: update sha256 hashes for new win32 download diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -410,7 +410,7 @@ 243cd0cc188a94c1f064f402ae72b8ba4303eb3137eac53c53826472b8005098 pypy3.6-v7.1.0-s390x.tar.bz2 faa81f469bb2a7cbd22c64f22d4b4ddc5a1f7c798d43b7919b629b932f9b1c6f pypy3.6-v7.1.0-src.tar.bz2 4858e7e8a0007bc3b381bd392208b28d30889a4e5a88a3c28e3d9dc4f25b654e pypy3.6-v7.1.0-src.zip -1383497f89e034e4124ea3e65979b0701b0296c06f4fec7082c8a39014a821f0 pypy3.6-v7.1.0-win32.zip +77a0576a3d518210467f0df2d0d9a1892c664566dc02f25d974c2dbc6b4749e7 pypy3.6-v7.1.0-win32.zip

pypy2.7-7.0.0 sha256:

diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -478,7 +478,7 @@
     243cd0cc188a94c1f064f402ae72b8ba4303eb3137eac53c53826472b8005098  pypy3.6-v7.1.0-s390x.tar.bz2
     faa81f469bb2a7cbd22c64f22d4b4ddc5a1f7c798d43b7919b629b932f9b1c6f  pypy3.6-v7.1.0-src.tar.bz2
     4858e7e8a0007bc3b381bd392208b28d30889a4e5a88a3c28e3d9dc4f25b654e  pypy3.6-v7.1.0-src.zip
-    1383497f89e034e4124ea3e65979b0701b0296c06f4fec7082c8a39014a821f0  pypy3.6-v7.1.0-win32.zip
+    77a0576a3d518210467f0df2d0d9a1892c664566dc02f25d974c2dbc6b4749e7  pypy3.6-v7.1.0-win32.zip
 
 pypy2.7-7.0.0 sha256::
 

From pypy.commits at gmail.com  Tue Mar 26 14:48:33 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 26 Mar 2019 11:48:33 -0700 (PDT)
Subject: [pypy-commit] pypy default: use append_utf8 in W_UnicodeBuilder
Message-ID: <5c9a7401.1c69fb81.b1029.576a@mx.google.com>

Author: Carl Friedrich Bolz-Tereick 
Branch: 
Changeset: r96350:bc9c696b8c95
Date: 2019-03-26 19:47 +0100
http://bitbucket.org/pypy/pypy/changeset/bc9c696b8c95/

Log:	use append_utf8 in W_UnicodeBuilder

diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py
--- a/pypy/module/__pypy__/interp_builders.py
+++ b/pypy/module/__pypy__/interp_builders.py
@@ -65,9 +65,12 @@
         return W_UnicodeBuilder(space, 3 * size)
 
     def descr_append(self, space, w_s):
-        w_unicode = W_UnicodeObject.convert_arg_to_w_unicode(space, w_s)
-        s = space.utf8_w(w_unicode)
-        self.builder.append(s)
+        if isinstance(w_s, W_UnicodeObject):
+            self.builder.append_utf8(w_s._utf8, w_s._len())
+        else:
+            w_unicode = W_UnicodeObject.convert_arg_to_w_unicode(space, w_s)
+            s = space.utf8_w(w_unicode)
+            self.builder.append(s)
 
     @unwrap_spec(start=int, end=int)
     def descr_append_slice(self, space, w_s, start, end):
diff --git a/pypy/module/__pypy__/test/test_builders.py b/pypy/module/__pypy__/test/test_builders.py
--- a/pypy/module/__pypy__/test/test_builders.py
+++ b/pypy/module/__pypy__/test/test_builders.py
@@ -1,14 +1,16 @@
+# -*- encoding: utf-8 -*-
+
 class AppTestBuilders(object):
     spaceconfig = dict(usemodules=['__pypy__'])
 
     def test_simple(self):
         from __pypy__.builders import UnicodeBuilder
         b = UnicodeBuilder()
-        b.append(u"abc")
+        b.append(u"abcä")
         b.append(u"123")
         b.append(u"1")
         s = b.build()
-        assert s == u"abc1231"
+        assert s == u"abcä1231"
         assert type(s) is unicode
         assert b.build() == s
         b.append(u"123")

From pypy.commits at gmail.com  Tue Mar 26 15:22:39 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 26 Mar 2019 12:22:39 -0700 (PDT)
Subject: [pypy-commit] pypy default: add missing check (fixes crash in scipy
 tests)
Message-ID: <5c9a7bff.1c69fb81.45b91.c775@mx.google.com>

Author: Matti Picus 
Branch: 
Changeset: r96351:66a0b0d41728
Date: 2019-03-26 21:09 +0200
http://bitbucket.org/pypy/pypy/changeset/66a0b0d41728/

Log:	add missing check (fixes crash in scipy tests)

diff --git a/pypy/module/zlib/interp_zlib.py b/pypy/module/zlib/interp_zlib.py
--- a/pypy/module/zlib/interp_zlib.py
+++ b/pypy/module/zlib/interp_zlib.py
@@ -351,6 +351,9 @@
             if length <= 0:
                 raise oefmt(space.w_ValueError,
                             "length must be greater than zero")
+        if not self.stream:
+            raise zlib_error(space,
+                             "compressor object already flushed")
         data = self.unconsumed_tail
         try:
             self.lock()
diff --git a/pypy/module/zlib/test/test_zlib.py b/pypy/module/zlib/test/test_zlib.py
--- a/pypy/module/zlib/test/test_zlib.py
+++ b/pypy/module/zlib/test/test_zlib.py
@@ -356,3 +356,11 @@
         compressor = self.zlib.compressobj()
         compressor.flush()
         raises(ValueError, compressor.copy)
+
+    def test_double_flush(self):
+        import zlib
+        x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E'  # 'foo'
+        dco = zlib.decompressobj()
+        dco.decompress(x)
+        dco.flush()
+        raises(self.zlib.error, dco.flush)

From pypy.commits at gmail.com  Tue Mar 26 15:22:42 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 26 Mar 2019 12:22:42 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge default into branch
Message-ID: <5c9a7c02.1c69fb81.d7b27.2e86@mx.google.com>

Author: Matti Picus 
Branch: py3.6
Changeset: r96352:50f4f89a4572
Date: 2019-03-26 21:16 +0200
http://bitbucket.org/pypy/pypy/changeset/50f4f89a4572/

Log:	merge default into branch

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -67,3 +67,11 @@
 928a4f70d3de7d17449456946154c5da6e600162 release-pypy3.5-v7.0.0
 dab365a465140aa79a5f3ba4db784c4af4d5c195 release-pypy3.6-v7.0.0
 fb40f7a5524c77b80e6c468e087d621610137261 release-pypy3.6-v7.0.0
+990cef41fe11e5d46b019a46aa956ff46ea1a234 release-pypy2.7-v7.1.0
+bb0d05b190b9c579f0c889a368636e14f6205bab release-pypy3.6-v7.1.0
+bb0d05b190b9c579f0c889a368636e14f6205bab release-pypy3.6-v7.1.0
+6fd188f8f903b7555920adf7d5e7fe21db1bd593 release-pypy3.6-v7.1.0
+6fd188f8f903b7555920adf7d5e7fe21db1bd593 release-pypy3.6-v7.1.0
+7a2e437acfceafe2665b23b1394dc6c66add3b89 release-pypy3.6-v7.1.0
+7a2e437acfceafe2665b23b1394dc6c66add3b89 release-pypy3.6-v7.1.0
+de061d87e39c7df4e436974096d7982c676a859d release-pypy3.6-v7.1.0
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -495,6 +495,9 @@
 * SyntaxError_ s try harder to give details about the cause of the failure, so
   the error messages are not the same as in CPython
 
+* Dictionaries and sets are ordered on PyPy.  On CPython < 3.6 they are not;
+  on CPython >= 3.6 dictionaries (but not sets) are ordered.
+
 
 .. _extension-modules:
 
diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -40,11 +40,11 @@
   $ hg up -r default
   $ # edit the version to e.g. 7.0.0-final
   $ hg ci
-  $ hg branch release-pypy2.7-7.x && hg ci
+  $ hg branch release-pypy2.7-v7.x && hg ci
   $ hg up -r default
   $ # edit the version to 7.1.0-alpha0
   $ hg ci
-  $ hg up -r release-pypy2.7-7.x
+  $ hg up -r release-pypy2.7-v7.x
   $ hg merge default
   $ # edit the version to AGAIN 7.0.0-final
   $ hg ci
@@ -53,11 +53,11 @@
 
   $ hg up -r py3.5
   $ hg merge default # this brings the version fo 7.1.0-alpha0
-  $ hg branch release-pypy3.5-7.x
+  $ hg branch release-pypy3.5-v7.x
   $ # edit the version to 7.0.0-final
   $ hg ci
   $ hg up -r py3.5
-  $ hg merge release-pypy3.5-7.x
+  $ hg merge release-pypy3.5-v7.x
   $ # edit the version to 7.1.0-alpha0
   $ hg ci
 
@@ -109,9 +109,11 @@
   * add a tag on the pypy/jitviewer repo that corresponds to pypy release, so
     that the source tarball can be produced in the next steps
 
-  * download the builds, repackage binaries. Tag the release version
-    and download and repackage source from bitbucket. You may find it
-    convenient to use the ``repackage.sh`` script in pypy/tool/release to do this. 
+  * download the builds, repackage binaries. Tag the release-candidate version
+    (it is important to mark this as a candidate since usually at least two
+    tries are needed to complete the process) and download and repackage source
+    from bitbucket. You may find it convenient to use the ``repackage.sh``
+    script in pypy/tool/release to do this. 
 
     Otherwise repackage and upload source "-src.tar.bz2" to bitbucket
     and to cobra, as some packagers prefer a clearly labeled source package
@@ -135,3 +137,5 @@
 
   * add a tag on the codespeed web site that corresponds to pypy release
   * revise versioning at https://readthedocs.org/projects/pypy
+  * tag the final release(s) with appropriate tags
+
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -5,7 +5,7 @@
 ----------------
 
 We are happy to discuss ideas around the PyPy ecosystem.
-If you are interested in palying with RPython or PyPy, or have a new idea not
+If you are interested in playing with RPython or PyPy, or have a new idea not
 mentioned here please join us on irc, channel #pypy (freenode). If you are unsure,
 but still think that you can make a valuable contribution to PyPy, dont
 hesitate to contact us on #pypy or on our mailing list. Here are some ideas
diff --git a/pypy/doc/release-v7.1.0.rst b/pypy/doc/release-v7.1.0.rst
--- a/pypy/doc/release-v7.1.0.rst
+++ b/pypy/doc/release-v7.1.0.rst
@@ -24,6 +24,10 @@
 Until we can work with downstream providers to distribute builds with PyPy, we
 have made packages for some common packages `available as wheels`_.
 
+The `CFFI`_ backend has been updated to version 1.12.2. We recommend using CFFI
+rather than c-extensions to interact with C, and `cppyy`_ for interacting with
+C++ code.
+
 As always, this release is 100% compatible with the previous one and fixed
 several issues and bugs raised by the growing community of PyPy users.
 We strongly recommend updating.
@@ -31,7 +35,7 @@
 The PyPy3.6 release is still not production quality so your mileage may vary.
 There are open issues with incomplete compatibility and c-extension support.
 
-You can download the v7.0 releases here:
+You can download the v7.1 releases here:
 
     http://pypy.org/download.html
 
@@ -47,7 +51,7 @@
 .. _`PyPy`: index.html
 .. _`RPython`: https://rpython.readthedocs.org
 .. _`help`: project-ideas.html
-.. _`cffi`: http://cffi.readthedocs.io
+.. _`CFFI`: http://cffi.readthedocs.io
 .. _`cppyy`: https://cppyy.readthedocs.io
 .. _`available as wheels`: https://github.com/antocuni/pypy-wheels
 
@@ -61,7 +65,7 @@
 We also welcome developers of other `dynamic languages`_ to see what RPython
 can do for them.
 
-The PyPy release supports:
+This PyPy release supports:
 
   * **x86** machines on most common operating systems
     (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
@@ -71,7 +75,8 @@
   * **s390x** running Linux
 
 Unfortunately at the moment of writing our ARM buildbots are out of service,
-so for now we are **not** releasing any binary for the ARM architecture.
+so for now we are **not** releasing any binary for the ARM architecture,
+although PyPy does support ARM 32 bit processors.
 
 .. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
 .. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html
@@ -80,5 +85,37 @@
 Changelog
 =========
 
-If not specified, the changes are shared across versions
+Changes shared across versions
 
+* Use utf8 internally to represent unicode, with the goal of never using
+  rpython-level unicode
+* Update ``cffi`` to 1.12.2
+* Improve performance of ``long`` operations where one of the operands fits
+  into an ``int``
+* Since _ctypes is implemented in pure python over libffi, add interfaces and
+  methods to support the buffer interface from python. Specifically, add a
+  ``__pypy__.newmemoryview`` function to create a memoryview and extend the use
+  of the PyPy-specific ``__buffer__`` class method. This enables better
+  buffer sharing between ctypes and NumPy.
+* Add copying to zlib
+* Improve register allocation in the JIT by using better heuristics
+* Include ```` on Gnu/Hurd
+* Mostly for completeness sake: support for ``rlib.jit.promote_unicode``, which
+  behaves like ``promote_string``, but for rpython unicode objects
+* Correctly initialize the ``d_type`` and ``d_name`` members of builtin
+  descriptors to fix a segfault related to classmethods in Cython
+* Expand documentation of ``__pypy_`` module
+
+C-API (cpyext) improvements shared across versions
+
+* Move PyTuple_Type.tp_new to C
+* Call internal methods from ``PyDict_XXXItem()`` instead of going through
+  dunder methods (CPython cpyext compatibility)
+
+Python 3.6 only
+
+* Support for os.PathLike in the posix module
+* Update ``idellib`` for 3.6.1
+* Make ``BUILD_CONST_KEY_MAP`` JIT-friendly
+* Adapt code that optimizes ``sys.exc_info()`` to wordcode
+* Fix annotation bug found by ``attrs``
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -3,4 +3,9 @@
 ==========================
 
 .. this is a revision shortly after release-pypy-7.1.0
-.. startrev: 78914a03cf95
+.. startrev: d3aefbf6dae7
+
+.. branch: Twirrim/minor-typo-fix-1553456951526
+
+Fix typo
+
diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py
--- a/pypy/module/__pypy__/interp_builders.py
+++ b/pypy/module/__pypy__/interp_builders.py
@@ -64,9 +64,12 @@
         return W_UnicodeBuilder(space, 3 * size)
 
     def descr_append(self, space, w_s):
-        w_unicode = W_UnicodeObject.convert_arg_to_w_unicode(space, w_s)
-        s = space.utf8_w(w_unicode)
-        self.builder.append(s)
+        if isinstance(w_s, W_UnicodeObject):
+            self.builder.append_utf8(w_s._utf8, w_s._len())
+        else:
+            w_unicode = W_UnicodeObject.convert_arg_to_w_unicode(space, w_s)
+            s = space.utf8_w(w_unicode)
+            self.builder.append(s)
 
     @unwrap_spec(start=int, end=int)
     def descr_append_slice(self, space, w_s, start, end):
diff --git a/pypy/module/__pypy__/test/test_builders.py b/pypy/module/__pypy__/test/test_builders.py
--- a/pypy/module/__pypy__/test/test_builders.py
+++ b/pypy/module/__pypy__/test/test_builders.py
@@ -1,14 +1,16 @@
+# -*- encoding: utf-8 -*-
+
 class AppTestBuilders(object):
     spaceconfig = dict(usemodules=['__pypy__'])
 
     def test_simple(self):
         from __pypy__.builders import StringBuilder
         b = StringBuilder()
-        b.append(u"abc")
+        b.append(u"abcä")
         b.append(u"123")
         b.append(u"1")
         s = b.build()
-        assert s == u"abc1231"
+        assert s == u"abcä1231"
         assert b.build() == s
         b.append(u"123")
         assert b.build() == s + u"123"
diff --git a/pypy/module/zlib/interp_zlib.py b/pypy/module/zlib/interp_zlib.py
--- a/pypy/module/zlib/interp_zlib.py
+++ b/pypy/module/zlib/interp_zlib.py
@@ -343,6 +343,9 @@
             if length <= 0:
                 raise oefmt(space.w_ValueError,
                             "length must be greater than zero")
+        if not self.stream:
+            raise zlib_error(space,
+                             "compressor object already flushed")
         data = self.unconsumed_tail
         try:
             self.lock()
diff --git a/pypy/module/zlib/test/test_zlib.py b/pypy/module/zlib/test/test_zlib.py
--- a/pypy/module/zlib/test/test_zlib.py
+++ b/pypy/module/zlib/test/test_zlib.py
@@ -414,3 +414,11 @@
         compressor = self.zlib.compressobj()
         compressor.flush()
         raises(ValueError, compressor.copy)
+
+    def test_double_flush(self):
+        import zlib
+        x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E'  # 'foo'
+        dco = zlib.decompressobj()
+        dco.decompress(x)
+        dco.flush()
+        raises(self.zlib.error, dco.flush)
diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,11 +1,15 @@
 # Edit these appropriately before running this script
-pmaj=2  # python main version
+pmaj=2  # python main version: 2 or 3
 pmin=7  # python minor version
+exe=pypy3 # pypy3 or pypy
 maj=7
-min=0
+min=1
 rev=0
-branchname=release-pypy$pmaj.$pmin-$maj.x # ==OR== release-$maj.x  # ==OR== release-$maj.$min.x
-tagname=release-pypy$pmaj.$pmin-v$maj.$min.$rev  # ==OR== release-$maj.$min
+
+
+branchname=release-pypy$pmaj.$pmin-v$maj.x # ==OR== release-v$maj.x  # ==OR== release-v$maj.$min.x
+tagname=release-candidate-pypy$pmaj.$pmin-v$maj.$min.$rev  # ==OR== release-$maj.$min
+# tagname=release-pypy$pmaj.$pmin-v$maj.$min.$rev  # ==OR== release-$maj.$min
 
 echo checking hg log -r $branchname
 hg log -r $branchname || exit 1
@@ -23,6 +27,7 @@
 
 # Download latest builds from the buildmaster, rename the top
 # level directory, and repackage ready to be uploaded to bitbucket
+actual_ver=xxxxxxxxxxxxxxx
 for plat in linux linux64 osx64 s390x # linux-armhf-raspbian linux-armel
   do
     echo downloading package for $plat
@@ -49,11 +54,24 @@
         plat_final=linux32
     fi
     mv pypy-c-jit-*-$plat $rel-$plat_final
+    # TODO: automate the platform choice or move it to the head of the file
+    if [ $plat_final == linux64 ]
+    then
+        actual_ver=`$rel-$plat_final/bin/$exe -c "import sys; print('.'.join([str(x) for x in sys.pypy_version_info[:2]]))"`
+    fi
     echo packaging $plat_final
     tar --owner=root --group=root --numeric-owner -cjf $rel-$plat_final.tar.bz2 $rel-$plat_final
     rm -rf $rel-$plat_final
   done
-
+if [ "$actual_ver" != "$maj.$min" ]
+then
+    echo xxxxxxxxxxxxxxxxxxxxxx
+    echo version mismatch, expected $maj.$min, got $actual_ver
+    echo xxxxxxxxxxxxxxxxxxxxxx
+    exit -1
+    rm -rf $rel-$plat_final
+    continue
+fi
 plat=win32
 if wget http://buildbot.pypy.org/nightly/$branchname/pypy-c-jit-latest-$plat.zip
 then

From pypy.commits at gmail.com  Tue Mar 26 15:23:52 2019
From: pypy.commits at gmail.com (mattip)
Date: Tue, 26 Mar 2019 12:23:52 -0700 (PDT)
Subject: [pypy-commit] pypy winoverlapped: merge py3.6 into branch
Message-ID: <5c9a7c48.1c69fb81.f0aa2.6e73@mx.google.com>

Author: Matti Picus 
Branch: winoverlapped
Changeset: r96353:c2dba4448ebf
Date: 2019-03-26 21:23 +0200
http://bitbucket.org/pypy/pypy/changeset/c2dba4448ebf/

Log:	merge py3.6 into branch

diff too long, truncating to 2000 out of 2299 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -67,3 +67,11 @@
 928a4f70d3de7d17449456946154c5da6e600162 release-pypy3.5-v7.0.0
 dab365a465140aa79a5f3ba4db784c4af4d5c195 release-pypy3.6-v7.0.0
 fb40f7a5524c77b80e6c468e087d621610137261 release-pypy3.6-v7.0.0
+990cef41fe11e5d46b019a46aa956ff46ea1a234 release-pypy2.7-v7.1.0
+bb0d05b190b9c579f0c889a368636e14f6205bab release-pypy3.6-v7.1.0
+bb0d05b190b9c579f0c889a368636e14f6205bab release-pypy3.6-v7.1.0
+6fd188f8f903b7555920adf7d5e7fe21db1bd593 release-pypy3.6-v7.1.0
+6fd188f8f903b7555920adf7d5e7fe21db1bd593 release-pypy3.6-v7.1.0
+7a2e437acfceafe2665b23b1394dc6c66add3b89 release-pypy3.6-v7.1.0
+7a2e437acfceafe2665b23b1394dc6c66add3b89 release-pypy3.6-v7.1.0
+de061d87e39c7df4e436974096d7982c676a859d release-pypy3.6-v7.1.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -123,7 +123,9 @@
   Wenzhu Man
   Konstantin Lopuhin
   John Witulski
+  Stefan Beyer
   Jeremy Thurgood
+  Andrew Lawrence
   Greg Price
   Ivan Sichmann Freitas
   Dario Bertini
@@ -134,7 +136,6 @@
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Stefan Beyer
   William Leslie
   Paweł Piotr Przeradowski
   marky1991
@@ -152,6 +153,7 @@
   Wanja Saatkamp
   Mike Blume
   Gerald Klix
+  Julian Berman
   Oscar Nierstrasz
   Rami Chowdhury
   Stefan H. Muller
@@ -174,6 +176,7 @@
   Anton Gulenko
   Sergey Matyunin
   Andrew Chambers
+  Łukasz Langa
   Nicolas Chauvat
   Andrew Durdin
   Ben Young
@@ -296,7 +299,6 @@
   Bobby Impollonia
   Roberto De Ioris
   Jeong YunWon
-  andrewjlawrence
   Christopher Armstrong
   Aaron Tubbs
   Vasantha Ganesh K
@@ -328,7 +330,6 @@
   Ben Darnell
   Juan Francisco Cantero Hurtado
   Godefroid Chappelle
-  Julian Berman
   Stephan Busemann
   Dan Colish
   timo
diff --git a/extra_tests/ctypes_tests/test_structures.py b/extra_tests/ctypes_tests/test_structures.py
--- a/extra_tests/ctypes_tests/test_structures.py
+++ b/extra_tests/ctypes_tests/test_structures.py
@@ -119,12 +119,15 @@
             ms.n = 0xff00
             return repr(ba[:])
 
+        nstruct = dostruct(Native)
         if sys.byteorder == 'little':
-            assert dostruct(Native) == dostruct(Little)
-            assert dostruct(Native) != dostruct(Big)
+            assert nstruct == dostruct(Little)
+            assert nstruct != dostruct(Big)
+            assert Big._fields_[0][1] is not i
         else:
-            assert dostruct(Native) == dostruct(Big)
-            assert dostruct(Native) != dostruct(Little)
+            assert nstruct == dostruct(Big)
+            assert nstruct != dostruct(Little)
+            assert Little._fields_[0][1] is not i
 
 def test_from_buffer_copy():
     from array import array
@@ -185,3 +188,20 @@
     assert sizeof(s) == 3 * sizeof(c_int)
     assert s.a == 4     # 256 + 4
     assert s.b == -123
+
+def test_memoryview():
+    class S(Structure):
+        _fields_ = [('a', c_int16),
+                    ('b', c_int16),
+                   ]
+
+    S3 = S * 3
+    c_array = (2 * S3)(
+        S3(S(a=0, b=1), S(a=2, b=3), S(a=4,  b=5)),
+        S3(S(a=6, b=7), S(a=8, b=9), S(a=10, b=11)),
+        )
+
+    mv = memoryview(c_array)
+    assert mv.format == 'T{'}
+swappedorder = {'little': '>', 'big': '<'}
+
+def get_format_str(typ):
+    if hasattr(typ, '_fields_'):
+        if hasattr(typ, '_swappedbytes_'):
+            bo = swappedorder[sys.byteorder]
+        else:
+            bo = byteorder[sys.byteorder]
+        flds = []
+        for name, obj in typ._fields_:
+            # Trim off the leading '<' or '>'
+            ch = get_format_str(obj)[1:]
+            if (ch) == 'B':
+                flds.append(byteorder[sys.byteorder])
+            else:
+                flds.append(bo)
+            flds.append(ch)
+            flds.append(':')
+            flds.append(name)
+            flds.append(':')
+        return 'T{' + ''.join(flds) + '}'
+    elif hasattr(typ, '_type_'):
+        ch = typ._type_
+        return byteorder[sys.byteorder] + ch
+    else:
+        raise ValueError('cannot get format string for %r' % typ)
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -2,8 +2,15 @@
 from _rawffi import alt as _ffi
 import sys
 
-try: from __pypy__ import builtinify
-except ImportError: builtinify = lambda f: f
+try:
+    from __pypy__ import builtinify
+except ImportError:
+    builtinify = lambda f: f
+
+try:
+    from __pypy__.bufferable import bufferable
+except ImportError:
+    bufferable = object
 
 keepalive_key = str # XXX fix this when provided with test
 
@@ -64,7 +71,7 @@
         'resbuffer' is a _rawffi array of length 1 containing the value,
         and this returns a general Python object that corresponds.
         """
-        res = object.__new__(self)
+        res = bufferable.__new__(self)
         res.__class__ = self
         res.__dict__['_buffer'] = resbuffer
         if base is not None:
@@ -148,7 +155,7 @@
     def __ne__(self, other):
         return self._obj != other
 
-class _CData(object, metaclass=_CDataMeta):
+class _CData(bufferable, metaclass=_CDataMeta):
     """ The most basic object for all ctypes types
     """
     _objects = None
diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -7,8 +7,7 @@
 from _ctypes.array import Array, array_get_slice_params, array_slice_getitem,\
      array_slice_setitem
 
-try: from __pypy__ import builtinify
-except ImportError: builtinify = lambda f: f
+from __pypy__ import builtinify, newmemoryview
 
 # This cache maps types to pointers to them.
 _pointer_type_cache = {}
@@ -134,6 +133,9 @@
     def _as_ffi_pointer_(self, ffitype):
         return as_ffi_pointer(self, ffitype)
 
+    def __buffer__(self, flags):
+        mv = memoryview(self.getcontents())
+        return newmemoryview(mv, mv.itemsize, '&' + mv.format, mv.shape)
 
 def _cast_addr(obj, _, tp):
     if not (isinstance(tp, _CDataMeta) and tp._is_pointer_like()):
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -2,9 +2,9 @@
 import _rawffi
 from _ctypes.basics import _CData, _CDataMeta, keepalive_key,\
      store_reference, ensure_objects, CArgObject
-from _ctypes.array import Array
+from _ctypes.array import Array, get_format_str
 from _ctypes.pointer import _Pointer
-import inspect
+import inspect, __pypy__
 
 
 def names_and_fields(self, _fields_, superclass, anonymous_fields=None):
@@ -176,6 +176,11 @@
 class StructOrUnionMeta(_CDataMeta):
     def __new__(self, name, cls, typedict):
         res = type.__new__(self, name, cls, typedict)
+        if hasattr(res, '_swappedbytes_') and '_fields_' in typedict:
+            # Activate the stdlib ctypes._swapped_meta.__setattr__ to convert fields
+            tmp = res._fields_
+            delattr(res, '_fields_')
+            setattr(res, '_fields_', tmp)
         if "_abstract_" in typedict:
             return res
         cls = cls or (object,)
@@ -253,17 +258,7 @@
                                          or cls is union.Union):
             raise TypeError("abstract class")
         if hasattr(cls, '_swappedbytes_'):
-            fields = [None] * len(cls._fields_)
-            for i in range(len(cls._fields_)):
-                if cls._fields_[i][1] == cls._fields_[i][1].__dict__.get('__ctype_be__', None):
-                    swapped = cls._fields_[i][1].__dict__.get('__ctype_le__', cls._fields_[i][1])
-                else:
-                    swapped = cls._fields_[i][1].__dict__.get('__ctype_be__', cls._fields_[i][1])
-                if len(cls._fields_[i]) < 3:
-                    fields[i] = (cls._fields_[i][0], swapped)
-                else:
-                    fields[i] = (cls._fields_[i][0], swapped, cls._fields_[i][2])
-            names_and_fields(cls, fields, _CData, cls.__dict__.get('_anonymous_', None))
+            names_and_fields(cls, cls._fields_, _CData, cls.__dict__.get('_anonymous_', None))
         self = super(_CData, cls).__new__(cls)
         if hasattr(cls, '_ffistruct_'):
             self.__dict__['_buffer'] = self._ffistruct_(autofree=True)
@@ -303,6 +298,10 @@
     def _to_ffi_param(self):
         return self._buffer
 
+    def __buffer__(self, flags):
+        fmt = get_format_str(self)
+        itemsize = type(self)._sizeofinstances() 
+        return __pypy__.newmemoryview(memoryview(self._buffer), itemsize, fmt)
 
 class StructureMeta(StructOrUnionMeta):
     _is_union = False
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.12.1
+Version: 1.12.2
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -5,8 +5,8 @@
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 from .error import PkgConfigError
 
-__version__ = "1.12.1"
-__version_info__ = (1, 12, 1)
+__version__ = "1.12.2"
+__version_info__ = (1, 12, 2)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -221,7 +221,7 @@
 
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.12.1"
+                               "\ncompiled with cffi version: 1.12.2"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst
--- a/pypy/doc/__pypy__-module.rst
+++ b/pypy/doc/__pypy__-module.rst
@@ -1,26 +1,186 @@
-.. comment: this document is very incomplete, should we generate
-            it automatically?
+.. comment: this document may get out of synch with the code, but to generate
+    it automatically we would need to use pypy to run sphinx-build
 
 The ``__pypy__`` module
 =======================
 
 The ``__pypy__`` module is the main entry point to special features provided
-by PyPy's standard interpreter. Its content depends on :doc:`configuration options `
-which may add new functionality and functions whose existence or non-existence
-indicates the presence of such features.
-
+by PyPy's standard interpreter. Its content depends on :doc:`configuration
+options ` which may add new functionality and functions whose
+existence or non-existence indicates the presence of such features. These are
+generally used for compatibility when writing pure python modules that in
+CPython are written in C. Not available in CPython, and so must be used inside a
+``if platform.python_implementation == 'PyPy'`` block or otherwise hidden from
+the CPython interpreter.
 
 Generally available functionality
 ---------------------------------
 
- - ``internal_repr(obj)``: return the interpreter-level representation of an
-   object.
- - ``bytebuffer(length)``: return a new read-write buffer of the given length.
-   It works like a simplified array of characters (actually, depending on the
-   configuration the ``array`` module internally uses this).
- - ``attach_gdb()``: start a GDB at the interpreter-level (or a PDB before translation).
+  - ``internal_repr(obj)``: return the interpreter-level representation of an
+    object.
+  - ``bytebuffer(length)``: return a new read-write buffer of the given length.
+    It works like a simplified array of characters (actually, depending on the
+    configuration the ``array`` module internally uses this).
 
+  - ``attach_gdb()``: start a GDB at the interpreter-level (or a PDB before translation).
 
+ - ``newmemoryview(buffer, itemsize, format, shape=None, strides=None)``:
+   create a `memoryview` instance with the data from ``buffer`` and the
+   specified itemsize, format, and optional shape and strides.
+
+ - ``bufferable``: a base class that provides a ``__buffer__(self, flags)``
+   method for subclasses to override. This method should return a memoryview
+   instance of the class instance. It is called by the C-API's ``tp_as_buffer.
+   bf_getbuffer``.
+
+  - ``builtinify(func)``: To implement at app-level modules that are, in CPython,
+    implemented in C: this decorator protects a function from being ever bound
+    like a method.  Useful because some tests do things like put a "built-in"
+    function on a class and access it via the instance.
+
+  - ``hidden_applevel(func)``: Decorator that hides a function's frame from
+    app-level
+
+  - ``get_hidden_tb()``: Return the traceback of the current exception being
+    handled by a frame hidden from applevel.
+
+  - ``lookup_special(obj, meth)``: Lookup up a special method on an object.
+  - ``do_what_I_mean``
+
+  - ``resizelist_hint(...)``: Reallocate the underlying storage of the argument
+    list to sizehint
+
+  - ``newlist_hint(...)``: Create a new empty list that has an underlying
+    storage of length sizehint
+
+  - ``add_memory_pressure(bytes)``: Add memory pressure of estimate bytes.
+    Useful when calling a C function that internally allocates a big chunk of
+    memory. This instructs the GC to garbage collect sooner than it would
+    otherwise.
+
+  - ``newdict(type)``: Create a normal dict with a special implementation
+    strategy. ``type`` is a string and can be:
+
+    * ``"module"`` - equivalent to ``some_module.__dict__``
+
+    * ``"instance"`` - equivalent to an instance dict with a not-changing-much
+      set of keys
+
+    * ``"kwargs"`` - keyword args dict equivalent of what you get from
+      ``**kwargs`` in a function, optimized for passing around
+
+    * ``"strdict"`` - string-key only dict. This one should be chosen
+      automatically
+
+  - ``reversed_dict``: Enumerate the keys in a dictionary object in reversed
+    order.  This is a ``__pypy__`` function instead of being simply done by
+    calling reversed(), for CPython compatibility: dictionaries are ordered in
+    PyPY but not in Cpython2.7.  You should use the collections.OrderedDict
+    class for cases where ordering is important. That class implements
+    ``__reversed__`` by calling __pypy__.reversed_dict()
+
+  - ``dict_popitem_first``: Interp-level implementation of
+    ``OrderedDict.popitem(last=False)``.
+
+  - ``delitem_if_value_is`` Atomic equivalent to: ``if dict.get(key) is value:
+    del dict[key]``.
+
+    SPECIAL USE CASES ONLY!  Avoid using on dicts which are specialized,
+    e.g. to ``int`` or ``str`` keys, because it switches to the object
+    strategy. Also, the ``is`` operation is really pointer equality, so avoid
+    using it if ``value`` is an immutable object like ``int`` or ``str``.
+
+  - ``move_to_end``: Move the key in a dictionary object into the first or last
+    position. This is used in Python 3.x to implement ``OrderedDict.move_to_end()``.
+
+  - ``strategy(dict or list or set)``: Return the underlying strategy currently
+    used by the object
+
+  - ``specialized_zip_2_lists``
+  - ``locals_to_fast``
+  - ``set_code_callback``
+  - ``save_module_content_for_future_reload``
+  - ``decode_long``
+  - ``side_effects_ok``: For use with the reverse-debugger: this function
+    normally returns True, but will return False if we are evaluating a
+    debugging command like a watchpoint.  You are responsible for not doing any
+    side effect at all (including no caching) when evaluating watchpoints. This
+    function is meant to help a bit---you can write::
+
+        if not __pypy__.side_effects_ok():
+            skip the caching logic
+
+    inside getter methods or properties, to make them usable from
+    watchpoints.  Note that you need to re-run ``REVDB=.. pypy``
+    after changing the Python code.
+
+  - ``stack_almost_full``: Return True if the stack is more than 15/16th full.
+  - ``pyos_inputhook``: Call PyOS_InputHook() from the CPython C API
+  - ``os.real_getenv(...)`` gets OS environment variables skipping python code
+  - ``_pypydatetime`` provides base classes with correct C API interactions for
+    the pure-python ``datetime`` stdlib module
+
+Fast String Concatenation
+-------------------------
+Rather than in-place concatenation ``+=``, use these to enable fast, minimal
+copy, string building.
+
+  - ``builders.StringBuilder``
+  - ``builders.UnicodeBuilder``
+
+Interacting with the PyPy debug log
+------------------------------------
+
+The following functions can be used to write your own content to the
+:ref:`PYPYLOG `.
+
+  - ``debug_start(category, timestamp=False)``: open a new section; if
+    ``timestamp`` is ``True``, also return the timestamp which was written to
+    the log.
+
+  - ``debug_stop(category, timestamp=False)``: close a section opened by
+    ``debug_start``.
+
+  - ``debug_print(...)``: print arbitrary text to the log.
+
+  - ``debug_print_once(category, ...)``: equivalent to ``debug_start`` +
+    ``debug_print`` + ``debug_stop``.
+
+  - ``debug_flush``: flush the log.
+
+  - ``debug_read_timestamp()``: read the timestamp from the same timer used by
+    the log.
+
+  - ``debug_get_timestamp_unit()``: get the unit of the value returned by
+    ``debug_read_timestamp()``.
+
+
+Depending on the architecture and operating system, PyPy uses different ways
+to read timestamps, so the timestamps used in the log file are expressed in
+varying units. It is possible to know which by calling
+``debug_get_timestamp_unit()``, which can be one of the following values:
+
+``tsc``
+    The default on ``x86`` machines: timestamps are expressed in CPU ticks, as
+    read by the `Time Stamp Counter`_.
+
+``ns``
+    Timestamps are expressed in nanoseconds.
+
+``QueryPerformanceCounter``
+    On Windows, in case for some reason ``tsc`` is not available: timestamps
+    are read using the win API ``QueryPerformanceCounter()``.
+
+
+Unfortunately, there does not seem to be a reliable standard way for
+converting ``tsc`` ticks into nanoseconds, although in practice on modern CPUs
+it is enough to divide the ticks by the maximum nominal frequency of the CPU.
+For this reason, PyPy gives the raw value, and leaves the job of doing the
+conversion to external libraries.
+
+.. _`Time Stamp Counter`: https://en.wikipedia.org/wiki/Time_Stamp_Counter    
+    
+   
 Transparent Proxy Functionality
 -------------------------------
 
@@ -34,6 +194,30 @@
    its controller. Otherwise return None.
 
 
+Additional Clocks for Timing
+----------------------------
+The ``time`` submodule exposes the platform-dependent clock types such as
+``CLOCK_BOOTTIME``, ``CLOCK_MONOTONIC``, ``CLOCK_MONOTONIC_COARSE``,
+``CLOCK_MONOTONIC_RAW`` and two functions:
+
+  - ``clock_gettime(m)`` which returns the clock type time in seconds and
+  - ``clock_getres(m)`` which returns the clock resolution in seconds.
+
+Extended Signal Handling
+------------------------
+``thread.signals_enbaled`` is a context manager to use in non-main threads.
+    enables receiving signals in a "with" statement.  More precisely, if a
+    signal is received by the process, then the signal handler might be
+    called either in the main thread (as usual) or within another thread
+    that is within a "with signals_enabled:".  This other thread should be
+    ready to handle unexpected exceptions that the signal handler might
+    raise --- notably KeyboardInterrupt.
+
+Integer Operations with Overflow
+--------------------------------
+  - ``intop`` provides a module with integer operations that have
+    two-complement overflow behaviour instead of overflowing to longs
+
 Functionality available on py.py (not after translation)
 --------------------------------------------------------
 
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -71,9 +71,9 @@
 #    module/cpyext/include/patchlevel.h
 #
 # The short X.Y version.
-version = '7.1'
+version = '7.2'
 # The full version, including alpha/beta/rc tags.
-release = '7.1.0'
+release = '7.2.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -90,7 +90,9 @@
   Wenzhu Man
   Konstantin Lopuhin
   John Witulski
+  Stefan Beyer
   Jeremy Thurgood
+  Andrew Lawrence
   Greg Price
   Ivan Sichmann Freitas
   Dario Bertini
@@ -101,7 +103,6 @@
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Stefan Beyer
   William Leslie
   Paweł Piotr Przeradowski
   marky1991
@@ -119,6 +120,7 @@
   Wanja Saatkamp
   Mike Blume
   Gerald Klix
+  Julian Berman
   Oscar Nierstrasz
   Rami Chowdhury
   Stefan H. Muller
@@ -141,6 +143,7 @@
   Anton Gulenko
   Sergey Matyunin
   Andrew Chambers
+  Łukasz Langa
   Nicolas Chauvat
   Andrew Durdin
   Ben Young
@@ -263,7 +266,6 @@
   Bobby Impollonia
   Roberto De Ioris
   Jeong YunWon
-  andrewjlawrence
   Christopher Armstrong
   Aaron Tubbs
   Vasantha Ganesh K
@@ -295,7 +297,6 @@
   Ben Darnell
   Juan Francisco Cantero Hurtado
   Godefroid Chappelle
-  Julian Berman
   Stephan Busemann
   Dan Colish
   timo
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -495,6 +495,9 @@
 * SyntaxError_ s try harder to give details about the cause of the failure, so
   the error messages are not the same as in CPython
 
+* Dictionaries and sets are ordered on PyPy.  On CPython < 3.6 they are not;
+  on CPython >= 3.6 dictionaries (but not sets) are ordered.
+
 
 .. _extension-modules:
 
diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst
--- a/pypy/doc/gc_info.rst
+++ b/pypy/doc/gc_info.rst
@@ -203,7 +203,7 @@
 
 ``duration``
     The total time spent inside minor collections since the last hook
-    call. See below for more information on the unit.
+    call, in seconds.
 
 ``duration_min``
     The duration of the fastest minor collection since the last hook call.
@@ -265,30 +265,6 @@
 ``gc-collect-done`` is used only to give additional stats, but doesn't do any
 actual work.
 
-A note about the ``duration`` field: depending on the architecture and
-operating system, PyPy uses different ways to read timestamps, so ``duration``
-is expressed in varying units. It is possible to know which by calling
-``__pypy__.debug_get_timestamp_unit()``, which can be one of the following
-values:
-
-``tsc``
-    The default on ``x86`` machines: timestamps are expressed in CPU ticks, as
-    read by the `Time Stamp Counter`_.
-
-``ns``
-    Timestamps are expressed in nanoseconds.
-
-``QueryPerformanceCounter``
-    On Windows, in case for some reason ``tsc`` is not available: timestamps
-    are read using the win API ``QueryPerformanceCounter()``.
-
-
-Unfortunately, there does not seem to be a reliable standard way for
-converting ``tsc`` ticks into nanoseconds, although in practice on modern CPUs
-it is enough to divide the ticks by the maximum nominal frequency of the CPU.
-For this reason, PyPy gives the raw value, and leaves the job of doing the
-conversion to external libraries.
-
 Here is an example of GC hooks in use::
 
     import sys
@@ -321,8 +297,6 @@
         lst = [lst, 1, 2, 3]
 
 
-.. _`Time Stamp Counter`: https://en.wikipedia.org/wiki/Time_Stamp_Counter    
-    
 .. _minimark-environment-variables:
 
 Environment variables
diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -40,11 +40,11 @@
   $ hg up -r default
   $ # edit the version to e.g. 7.0.0-final
   $ hg ci
-  $ hg branch release-pypy2.7-7.x && hg ci
+  $ hg branch release-pypy2.7-v7.x && hg ci
   $ hg up -r default
   $ # edit the version to 7.1.0-alpha0
   $ hg ci
-  $ hg up -r release-pypy2.7-7.x
+  $ hg up -r release-pypy2.7-v7.x
   $ hg merge default
   $ # edit the version to AGAIN 7.0.0-final
   $ hg ci
@@ -53,11 +53,11 @@
 
   $ hg up -r py3.5
   $ hg merge default # this brings the version fo 7.1.0-alpha0
-  $ hg branch release-pypy3.5-7.x
+  $ hg branch release-pypy3.5-v7.x
   $ # edit the version to 7.0.0-final
   $ hg ci
   $ hg up -r py3.5
-  $ hg merge release-pypy3.5-7.x
+  $ hg merge release-pypy3.5-v7.x
   $ # edit the version to 7.1.0-alpha0
   $ hg ci
 
@@ -109,9 +109,11 @@
   * add a tag on the pypy/jitviewer repo that corresponds to pypy release, so
     that the source tarball can be produced in the next steps
 
-  * download the builds, repackage binaries. Tag the release version
-    and download and repackage source from bitbucket. You may find it
-    convenient to use the ``repackage.sh`` script in pypy/tool/release to do this. 
+  * download the builds, repackage binaries. Tag the release-candidate version
+    (it is important to mark this as a candidate since usually at least two
+    tries are needed to complete the process) and download and repackage source
+    from bitbucket. You may find it convenient to use the ``repackage.sh``
+    script in pypy/tool/release to do this. 
 
     Otherwise repackage and upload source "-src.tar.bz2" to bitbucket
     and to cobra, as some packagers prefer a clearly labeled source package
@@ -135,3 +137,5 @@
 
   * add a tag on the codespeed web site that corresponds to pypy release
   * revise versioning at https://readthedocs.org/projects/pypy
+  * tag the final release(s) with appropriate tags
+
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -1,11 +1,12 @@
 Historical release notes
 ========================
 
-CPython 2.7 compatible versions
--------------------------------
+Combined releases
+-----------------
 
 .. toctree::
 
+   release-v7.1.0.rst
    release-v7.0.0.rst
    release-v6.0.0.rst
    release-v5.10.1.rst
@@ -14,6 +15,12 @@
    release-v5.8.0.rst
    release-v5.7.1.rst
    release-v5.7.0.rst
+
+CPython 2.7 compatible versions
+-------------------------------
+
+.. toctree::
+
    release-pypy2.7-v5.6.0.rst
    release-pypy2.7-v5.4.1.rst
    release-pypy2.7-v5.4.0.rst
@@ -61,15 +68,6 @@
    release-0.7.0.rst
    release-0.6
 
-CPython 3.5 compatible versions
--------------------------------
-
-.. toctree::
-
-   release-v5.8.0.rst
-   release-v5.7.1.rst
-   release-v5.7.0.rst
-
 CPython 3.3 compatible versions
 -------------------------------
 
diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst
--- a/pypy/doc/man/pypy.1.rst
+++ b/pypy/doc/man/pypy.1.rst
@@ -99,6 +99,8 @@
     If set, equivalent to the ``-W`` option (warning control).
     The value should be a comma-separated list of ``-W`` parameters.
 
+.. _pypylog:
+
 ``PYPYLOG``
     If set to a non-empty value, enable logging, the format is:
 
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -5,7 +5,7 @@
 ----------------
 
 We are happy to discuss ideas around the PyPy ecosystem.
-If you are interested in palying with RPython or PyPy, or have a new idea not
+If you are interested in playing with RPython or PyPy, or have a new idea not
 mentioned here please join us on irc, channel #pypy (freenode). If you are unsure,
 but still think that you can make a valuable contribution to PyPy, dont
 hesitate to contact us on #pypy or on our mailing list. Here are some ideas
diff --git a/pypy/doc/release-v7.1.0.rst b/pypy/doc/release-v7.1.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-v7.1.0.rst
@@ -0,0 +1,121 @@
+=========================================
+PyPy v7.1.0: release of 2.7, and 3.6-beta
+=========================================
+
+The PyPy team is proud to release the version 7.1.0 of PyPy, which includes
+two different interpreters:
+
+  - PyPy2.7, which is an interpreter supporting the syntax and the features of
+    Python 2.7
+
+  - PyPy3.6-beta: this is the second official release of PyPy to support 3.6
+    features, although it is still considered beta quality.
+    
+The interpreters are based on much the same codebase, thus the double
+release.
+
+This release, coming fast on the heels of 7.0 in February, finally merges the
+internal refactoring of unicode representation as UTF-8. Removing the
+conversions from strings to unicode internally lead to a nice speed bump.
+
+We also improved the ability to use the buffer protocol with ctype structures
+and arrays.
+
+Until we can work with downstream providers to distribute builds with PyPy, we
+have made packages for some common packages `available as wheels`_.
+
+The `CFFI`_ backend has been updated to version 1.12.2. We recommend using CFFI
+rather than c-extensions to interact with C, and `cppyy`_ for interacting with
+C++ code.
+
+As always, this release is 100% compatible with the previous one and fixed
+several issues and bugs raised by the growing community of PyPy users.
+We strongly recommend updating.
+
+The PyPy3.6 release is still not production quality so your mileage may vary.
+There are open issues with incomplete compatibility and c-extension support.
+
+You can download the v7.1 releases here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project. If PyPy is not quite good enough for your needs, we are available for
+direct consulting work.
+
+We would also like to thank our contributors and encourage new people to join
+the project. PyPy has many layers and we need help with all of them: `PyPy`_
+and `RPython`_ documentation improvements, tweaking popular modules to run
+on pypy, or general `help`_ with making RPython's JIT even better.
+
+.. _`PyPy`: index.html
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`help`: project-ideas.html
+.. _`CFFI`: http://cffi.readthedocs.io
+.. _`cppyy`: https://cppyy.readthedocs.io
+.. _`available as wheels`: https://github.com/antocuni/pypy-wheels
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7, 3.6. It's fast (`PyPy and CPython 2.7.x`_ performance
+comparison) due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+This PyPy release supports:
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+Unfortunately at the moment of writing our ARM buildbots are out of service,
+so for now we are **not** releasing any binary for the ARM architecture,
+although PyPy does support ARM 32 bit processors.
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html
+
+
+Changelog
+=========
+
+Changes shared across versions
+
+* Use utf8 internally to represent unicode, with the goal of never using
+  rpython-level unicode
+* Update ``cffi`` to 1.12.2
+* Improve performance of ``long`` operations where one of the operands fits
+  into an ``int``
+* Since _ctypes is implemented in pure python over libffi, add interfaces and
+  methods to support the buffer interface from python. Specifically, add a
+  ``__pypy__.newmemoryview`` function to create a memoryview and extend the use
+  of the PyPy-specific ``__buffer__`` class method. This enables better
+  buffer sharing between ctypes and NumPy.
+* Add copying to zlib
+* Improve register allocation in the JIT by using better heuristics
+* Include ```` on Gnu/Hurd
+* Mostly for completeness sake: support for ``rlib.jit.promote_unicode``, which
+  behaves like ``promote_string``, but for rpython unicode objects
+* Correctly initialize the ``d_type`` and ``d_name`` members of builtin
+  descriptors to fix a segfault related to classmethods in Cython
+* Expand documentation of ``__pypy_`` module
+
+C-API (cpyext) improvements shared across versions
+
+* Move PyTuple_Type.tp_new to C
+* Call internal methods from ``PyDict_XXXItem()`` instead of going through
+  dunder methods (CPython cpyext compatibility)
+
+Python 3.6 only
+
+* Support for os.PathLike in the posix module
+* Update ``idellib`` for 3.6.1
+* Make ``BUILD_CONST_KEY_MAP`` JIT-friendly
+* Adapt code that optimizes ``sys.exc_info()`` to wordcode
+* Fix annotation bug found by ``attrs``
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
--- a/pypy/doc/tool/makecontributor.py
+++ b/pypy/doc/tool/makecontributor.py
@@ -1,4 +1,5 @@
 # NOTE: run this script with LANG=en_US.UTF-8
+# works with pip install mercurial==3.0
 
 import py
 import sys
@@ -89,6 +90,7 @@
     'Laurence Tratt': ['ltratt'],
     'Pieter Zieschang': ['pzieschang', 'p_zieschang at yahoo.de'],
     'John Witulski': ['witulski'],
+    'Andrew Lawrence': ['andrew.lawrence at siemens.com', 'andrewjlawrence'],
     }
 
 alias_map = {}
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,34 +1,11 @@
 ==========================
-What's new in PyPy2.7 7.0+
+What's new in PyPy2.7 7.1+
 ==========================
 
-.. this is a revision shortly after release-pypy-7.0.0
-.. startrev: 481c69f7d81f
+.. this is a revision shortly after release-pypy-7.1.0
+.. startrev: d3aefbf6dae7
 
-.. branch: zlib-copying-third-time-a-charm
+.. branch: Twirrim/minor-typo-fix-1553456951526
 
-Make sure zlib decompressobjs have their streams deallocated immediately
-on flush.
+Fix typo
 
-.. branch: zlib-copying-redux
-
-Fix calling copy on already-flushed compressobjs.
-
-
-
-.. branch: math-improvements
-
-Improve performance of long operations where one of the operands fits into
-an int.
-
-.. branch: regalloc-playground
-
-Improve register allocation in the JIT.
-
-.. branch: promote-unicode
-
-Implement rlib.jit.promote_unicode to complement promote_string
-
-.. branch: unicode-utf8
-
-Use utf8 internally to represent unicode, with the goal of never using rpython-level unicode
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-7.1.0.rst
copy from pypy/doc/whatsnew-head.rst
copy to pypy/doc/whatsnew-pypy2-7.1.0.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-pypy2-7.1.0.rst
@@ -32,3 +32,10 @@
 .. branch: unicode-utf8
 
 Use utf8 internally to represent unicode, with the goal of never using rpython-level unicode
+
+.. branch: newmemoryview-app-level
+
+Since _ctypes is implemented in pure python over libffi, add interfaces and
+methods to support the buffer interface from python. Specifically, add a
+``__pypy__.newmemoryview`` function to create a memoryview and extend the use
+of the PyPy-specific ``__buffer__`` class method.
\ No newline at end of file
diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-7.1.0.rst
copy from pypy/doc/whatsnew-pypy3-head.rst
copy to pypy/doc/whatsnew-pypy3-7.1.0.rst
diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -1,11 +1,10 @@
 ========================
-What's new in PyPy3 7.0+
+What's new in PyPy3 7.1+
 ========================
 
-.. this is the revision after release-pypy3.6-v7.0
-.. startrev: 33fe3b2cf186
+.. this is the revision after release-pypy3.6-v7.1
+.. startrev: d642a3c217cb
 
-.. branch: py3.5
+.. branch: zlib-make-py3-go-boom
 
-Merge in py.35 and use this branch as the primary pypy3 one
-
+Complain if you try to copy a flushed zlib decompress on py3
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -362,6 +362,8 @@
     valid so we're trying to either raise or pack stuff with error handler.
     The key difference is that this is call_may_force
     """
+    if errors is None:
+        errors = 'strict'
     slen = len(s)
     res = StringBuilder(slen)
     pos = 0
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -62,11 +62,18 @@
 class PyPyDateTime(MixedModule):
     appleveldefs = {}
     interpleveldefs = {
-        'dateinterop': 'interp_pypydatetime.W_DateTime_Date',
-        'timeinterop'    : 'interp_pypydatetime.W_DateTime_Time',
-        'deltainterop'   : 'interp_pypydatetime.W_DateTime_Delta',
+        'dateinterop'  : 'interp_pypydatetime.W_DateTime_Date',
+        'timeinterop'  : 'interp_pypydatetime.W_DateTime_Time',
+        'deltainterop' : 'interp_pypydatetime.W_DateTime_Delta',
     }
 
+class PyPyBufferable(MixedModule):
+    appleveldefs = {}
+    interpleveldefs = {
+        'bufferable': 'interp_buffer.W_Bufferable',
+    }
+        
+
 class Module(MixedModule):
     """ PyPy specific "magic" functions. A lot of them are experimental and
     subject to change, many are internal. """
@@ -111,6 +118,7 @@
         'fsencode'                  : 'interp_magic.fsencode',
         'fsdecode'                  : 'interp_magic.fsdecode',
         'pyos_inputhook'            : 'interp_magic.pyos_inputhook',
+        'newmemoryview'             : 'interp_buffer.newmemoryview',
     }
 
     submodules = {
@@ -120,6 +128,7 @@
         "intop": IntOpModule,
         "os": OsModule,
         '_pypydatetime': PyPyDateTime,
+        'bufferable': PyPyBufferable,
     }
 
     def setup_after_space_initialization(self):
diff --git a/pypy/module/__pypy__/interp_buffer.py b/pypy/module/__pypy__/interp_buffer.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__pypy__/interp_buffer.py
@@ -0,0 +1,100 @@
+#
+# An app-level interface to tp_as_buffer->bf_getbuffer.
+#
+
+from pypy.interpreter.error import oefmt
+from pypy.interpreter.gateway import unwrap_spec, interp2app
+from pypy.objspace.std.memoryobject import BufferViewND
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.typedef import TypeDef, generic_new_descr
+
+class W_Bufferable(W_Root):
+    def __init__(self, space):
+        pass
+
+    def descr_buffer(self, space, w_flags):
+        if type(self) is W_Bufferable:
+            raise oefmt(space.w_ValueError, "override __buffer__ in a subclass")
+        return space.call_method(self, '__buffer__', w_flags)
+
+    def readbuf_w(self, space):
+        mv = space.call_method(self, '__buffer__', space.newint(0))
+        return mv.buffer_w(space, 0).as_readbuf()
+
+W_Bufferable.typedef = TypeDef("Bufferable", None, None, 'read-write',
+    __doc__ = """a helper class for a app-level class (like _ctypes.Array)
+that want to support tp_as_buffer.bf_getbuffer via a __buffer__ method""",
+    __new__ = generic_new_descr(W_Bufferable),
+    __buffer__ = interp2app(W_Bufferable.descr_buffer),
+)
+
+ at unwrap_spec(itemsize=int, format='text')
+def newmemoryview(space, w_obj, itemsize, format, w_shape=None, w_strides=None):
+    '''
+    newmemoryview(buf, itemsize, format, shape=None, strides=None)
+    '''
+    if not space.isinstance_w(w_obj, space.w_memoryview):
+        raise oefmt(space.w_ValueError, "memoryview expected")
+    # minimal error checking
+    lgt = space.len_w(w_obj)
+    old_size = w_obj.getitemsize()
+    nbytes = lgt * old_size
+    if w_shape:
+        tot = 1
+        shape = []
+        for w_v in space.listview(w_shape):
+            v = space.int_w(w_v)
+            shape.append(v)
+            tot *= v
+        if tot * itemsize != nbytes:
+            raise oefmt(space.w_ValueError,
+                  "shape/itemsize %s/%d does not match obj len/itemsize %d/%d",
+                  str(shape), itemsize, lgt, old_size)
+    else:
+        if nbytes % itemsize != 0:
+            raise oefmt(space.w_ValueError,
+                  "itemsize %d does not match obj len/itemsize %d/%d",
+                  itemsize, lgt, old_size)
+        shape = [nbytes / itemsize,]
+    ndim = len(shape)
+    if w_strides:
+        strides = [] 
+        for w_v in space.listview(w_strides):
+            v = space.int_w(w_v)
+            strides.append(v)
+        if not w_shape and len(strides) != 1:
+            raise oefmt(space.w_ValueError,
+                  "strides must have one value if shape not provided")
+        if len(strides) != ndim:
+            raise oefmt(space.w_ValueError,
+                  "shape %s does not match strides %s",
+                  str(shape), str(strides))
+    else:
+        # start from the right, c-order layout
+        strides = [itemsize] * ndim
+        for v in range(ndim - 2, -1, -1):
+            strides[v] = strides[v + 1] * shape[v + 1]
+    # check that the strides are not too big
+    for i in range(ndim):
+        if strides[i] * shape[i] > nbytes:
+            raise oefmt(space.w_ValueError,
+                  "shape %s and strides %s exceed object size %d",
+                  shape, strides, nbytes)
+    view = space.buffer_w(w_obj, 0)
+    return space.newmemoryview(FormatBufferViewND(view, itemsize, format, ndim,
+                                                  shape, strides))
+
+class FormatBufferViewND(BufferViewND):
+    _immutable_ = True
+    _attrs_ = ['readonly', 'parent', 'ndim', 'shape', 'strides',
+               'format', 'itemsize']
+    def __init__(self, parent, itemsize, format, ndim, shape, strides):
+        BufferViewND.__init__(self, parent, ndim, shape, strides)
+        self.format = format
+        self.itemsize = itemsize
+
+    def getformat(self):
+        return self.format
+
+    def getitemsize(self):
+        return self.itemsize
diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py
--- a/pypy/module/__pypy__/interp_builders.py
+++ b/pypy/module/__pypy__/interp_builders.py
@@ -64,9 +64,12 @@
         return W_UnicodeBuilder(space, 3 * size)
 
     def descr_append(self, space, w_s):
-        w_unicode = W_UnicodeObject.convert_arg_to_w_unicode(space, w_s)
-        s = space.utf8_w(w_unicode)
-        self.builder.append(s)
+        if isinstance(w_s, W_UnicodeObject):
+            self.builder.append_utf8(w_s._utf8, w_s._len())
+        else:
+            w_unicode = W_UnicodeObject.convert_arg_to_w_unicode(space, w_s)
+            s = space.utf8_w(w_unicode)
+            self.builder.append(s)
 
     @unwrap_spec(start=int, end=int)
     def descr_append_slice(self, space, w_s, start, end):
diff --git a/pypy/module/__pypy__/test/test_builders.py b/pypy/module/__pypy__/test/test_builders.py
--- a/pypy/module/__pypy__/test/test_builders.py
+++ b/pypy/module/__pypy__/test/test_builders.py
@@ -1,14 +1,16 @@
+# -*- encoding: utf-8 -*-
+
 class AppTestBuilders(object):
     spaceconfig = dict(usemodules=['__pypy__'])
 
     def test_simple(self):
         from __pypy__.builders import StringBuilder
         b = StringBuilder()
-        b.append(u"abc")
+        b.append(u"abcä")
         b.append(u"123")
         b.append(u"1")
         s = b.build()
-        assert s == u"abc1231"
+        assert s == u"abcä1231"
         assert b.build() == s
         b.append(u"123")
         assert b.build() == s + u"123"
diff --git a/pypy/module/__pypy__/test/test_newmemoryview.py b/pypy/module/__pypy__/test/test_newmemoryview.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__pypy__/test/test_newmemoryview.py
@@ -0,0 +1,32 @@
+
+
+class AppTestMinimal:
+    spaceconfig = dict(usemodules=['__pypy__'])
+
+    def test_newmemoryview(self):
+        from __pypy__ import newmemoryview
+        b = bytearray(12)
+        # The format can be anything, we only verify shape, strides, and itemsize
+        m = newmemoryview(memoryview(b), 2, 'T{= 0
+        return max(size, optvarsize)
+
     def write_v(self, cdata, w_ob, optvarsize):
         # a special case for var-sized C99 arrays
         from pypy.module._cffi_backend import ctypearray
         ct = self.ctype
+        space = ct.space
         if isinstance(ct, ctypearray.W_CTypeArray) and ct.length < 0:
-            space = ct.space
             w_ob, varsizelength = ct.get_new_array_length(w_ob)
             if optvarsize != -1:
                 # in this mode, the only purpose of this function is to compute
                 # the real size of the structure from a var-sized C99 array
                 assert cdata == lltype.nullptr(rffi.CCHARP.TO)
-                itemsize = ct.ctitem.size
-                try:
-                    varsize = ovfcheck(itemsize * varsizelength)
-                    size = ovfcheck(self.offset + varsize)
-                except OverflowError:
-                    raise oefmt(space.w_OverflowError,
-                                "array size would overflow a ssize_t")
-                assert size >= 0
-                return max(size, optvarsize)
+                return self.add_varsize_length(space, ct.ctitem.size,
+                    varsizelength, optvarsize)
             # if 'value' was only an integer, get_new_array_length() returns
             # w_ob = space.w_None.  Detect if this was the case,
             # and if so, stop here, leaving the content uninitialized
@@ -267,6 +273,12 @@
         #
         if optvarsize == -1:
             self.write(cdata, w_ob)
+        elif (isinstance(ct, W_CTypeStructOrUnion) and ct._with_var_array and
+              not isinstance(w_ob, cdataobj.W_CData)):
+            subsize = ct.size
+            subsize = ct.convert_struct_from_object(
+                lltype.nullptr(rffi.CCHARP.TO), w_ob, subsize)
+            optvarsize = self.add_varsize_length(space, 1, subsize, optvarsize)
         return optvarsize
 
     def convert_bitfield_to_object(self, cdata):
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -368,6 +368,16 @@
                 raise oefmt(space.w_TypeError,
                             "field '%s.%s' has ctype '%s' of unknown size",
                             w_ctype.name, fname, ftype.name)
+        elif isinstance(ftype, ctypestruct.W_CTypeStructOrUnion):
+            ftype.force_lazy_struct()
+            # GCC (or maybe C99) accepts var-sized struct fields that are not
+            # the last field of a larger struct.  That's why there is no
+            # check here for "last field": we propagate the flag
+            # '_with_var_array' to any struct that contains either an open-
+            # ended array or another struct that recursively contains an
+            # open-ended array.
+            if ftype._with_var_array:
+                with_var_array = True
         #
         if is_union:
             boffset = 0         # reset each field at offset 0
@@ -419,7 +429,6 @@
                 # a nested anonymous struct or union
                 # note: it seems we only get here with ffi.verify()
                 srcfield2names = {}
-                ftype.force_lazy_struct()
                 for name, srcfld in ftype._fields_dict.items():
                     srcfield2names[srcfld] = name
                 for srcfld in ftype._fields_list:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.12.1", ("This test_c.py file is for testing a version"
+assert __version__ == "1.12.2", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -3441,6 +3441,15 @@
     assert p.a[1] == 20
     assert p.a[2] == 30
     assert p.a[3] == 0
+    #
+    # struct of struct of varsized array
+    BStruct2 = new_struct_type("bar")
+    complete_struct_or_union(BStruct2, [('head', BInt),
+                                        ('tail', BStruct)])
+    for i in range(2):   # try to detect heap overwrites
+        p = newp(new_pointer_type(BStruct2), [100, [200, list(range(50))]])
+        assert p.tail.y[49] == 49
+
 
 def test_struct_array_no_length_explicit_position():
     BInt = new_primitive_type("int")
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -304,7 +304,7 @@
         while pos < end:
             oc = ord(obj[pos])
             raw_unicode_escape_helper(builder, oc)
-            pos += 1 
+            pos += 1
         return space.newtuple([space.newtext(builder.build()), w_end])
     else:
         raise oefmt(space.w_TypeError,
@@ -526,7 +526,10 @@
 
 def _call_codec(space, w_coder, w_obj, action, encoding, errors):
     try:
-        w_res = space.call_function(w_coder, w_obj, space.newtext(errors))
+        if errors:
+            w_res = space.call_function(w_coder, w_obj, space.newtext(errors))
+        else:
+            w_res = space.call_function(w_coder, w_obj)
     except OperationError as operr:
         raise _wrap_codec_error(space, operr, action, encoding)
     if (not space.isinstance_w(w_res, space.w_tuple) or space.len_w(w_res) != 2):
@@ -558,8 +561,8 @@
     return w_err_handler
 
 
- at unwrap_spec(errors='text')
-def encode(space, w_obj, w_encoding=None, errors='strict'):
+ at unwrap_spec(encoding='text_or_none', errors='text_or_none')
+def encode(space, w_obj, encoding=None, errors=None):
     """encode(obj, [encoding[,errors]]) -> object
 
     Encodes obj using the codec registered for encoding. encoding defaults
@@ -569,20 +572,26 @@
     'xmlcharrefreplace' as well as any other name registered with
     codecs.register_error that can handle ValueErrors.
     """
-    if w_encoding is None:
+    if encoding is None:
         encoding = space.sys.defaultencoding
-    else:
-        encoding = space.text_w(w_encoding)
     w_encoder = space.getitem(lookup_codec(space, encoding), space.newint(0))
-    return _call_codec(space, w_encoder, w_obj, "encoding", encoding, errors)
+    w_retval =  _call_codec(space, w_encoder, w_obj, "encoding", encoding, errors)
+    if not space.isinstance_w(w_retval, space.w_bytes):
+        raise oefmt(space.w_TypeError,
+                    "'%s' encoder returned '%T' instead of 'bytes'; "
+                    "use codecs.encode() to encode to arbitrary types",
+                    encoding,
+                    w_retval)
+    return w_retval
 
 @unwrap_spec(errors='text_or_none')
 def readbuffer_encode(space, w_data, errors='strict'):
     s = space.getarg_w('s#', w_data)
     return space.newtuple([space.newbytes(s), space.newint(len(s))])
 
- at unwrap_spec(errors='text')
-def decode(space, w_obj, w_encoding=None, errors='strict'):
+ at unwrap_spec(encoding='text_or_none', errors='text_or_none')
+def decode(space, w_obj, encoding=None, errors=None):
+    from pypy.objspace.std.unicodeobject import W_UnicodeObject
     """decode(obj, [encoding[,errors]]) -> object
 
     Decodes obj using the codec registered for encoding. encoding defaults
@@ -592,12 +601,17 @@
     as well as any other name registered with codecs.register_error that is
     able to handle ValueErrors.
     """
-    if w_encoding is None:
+    if encoding is None:
         encoding = space.sys.defaultencoding
-    else:
-        encoding = space.text_w(w_encoding)
     w_decoder = space.getitem(lookup_codec(space, encoding), space.newint(1))
-    return _call_codec(space, w_decoder, w_obj, "decoding", encoding, errors)
+    w_retval = _call_codec(space, w_decoder, w_obj, "decoding", encoding, errors)
+    if not isinstance(w_retval, W_UnicodeObject):
+        raise oefmt(space.w_TypeError,
+                    "'%s' decoder returned '%T' instead of 'str'; "
+                    "use codecs.decode() to decode to arbitrary types",
+                    encoding,
+                    w_retval)
+    return w_retval
 
 @unwrap_spec(errors='text')
 def register_error(space, errors, w_handler):
@@ -633,20 +647,6 @@
                     "use %s to handle arbitrary codecs", encoding, action)
     return codec_info
 
-def encode_text(space, w_obj, encoding, errors):
-    if errors is None:
-        errors = 'strict'
-    w_encoder = space.getitem(
-        lookup_text_codec(space, "codecs.encode()", encoding), space.newint(0))
-    return _call_codec(space, w_encoder, w_obj, "encoding", encoding, errors)
-
-def decode_text(space, w_obj, encoding, errors):
-    if errors is None:
-        errors = 'strict'
-    w_decoder = space.getitem(
-        lookup_text_codec(space, "codecs.decode()", encoding), space.newint(1))
-    return _call_codec(space, w_decoder, w_obj, "decoding", encoding, errors)
-
 # ____________________________________________________________
 
 def _find_implementation(impl_name):
@@ -736,7 +736,7 @@
         result = unicodehelper.utf8_encode_utf_8(utf8, errors,
                      state.encode_error_handler, allow_surrogates=False)
     except unicodehelper.ErrorHandlerError as e:
-        raise oefmt(space.w_IndexError, 
+        raise oefmt(space.w_IndexError,
                    "position %d from error handler invalid, already encoded %d",
                     e.new,e.old)
 
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -696,6 +696,22 @@
         exc = raises(RuntimeError, u"hello".encode, "test.failingenc")
         assert exc.value == to_raise
 
+    def test_one_arg_encoder(self):
+        import _codecs
+        def search_function(encoding):
+            def encode_one(u):
+                return (b'foo', len(u))
+            def decode_one(u):
+                return (u'foo', len(u))
+            if encoding == 'onearg':
+                return (encode_one, decode_one, None, None)
+            return None
+        _codecs.register(search_function)
+        assert u"hello".encode("onearg") == b'foo'
+        assert b"hello".decode("onearg") == u'foo'
+        assert _codecs.encode(u"hello", "onearg") == b'foo'
+        assert _codecs.decode(b"hello", "onearg") == u'foo'
+
     def test_cpytest_decode(self):
         import codecs
         assert codecs.decode(b'\xe4\xf6\xfc', 'latin-1') == '\xe4\xf6\xfc'
diff --git a/pypy/module/_multiprocessing/test/test_semaphore.py b/pypy/module/_multiprocessing/test/test_semaphore.py
--- a/pypy/module/_multiprocessing/test/test_semaphore.py
+++ b/pypy/module/_multiprocessing/test/test_semaphore.py
@@ -11,7 +11,7 @@
                                    'binascii', 'struct', '_posixsubprocess'))
 
     if sys.platform == 'win32':
-        spaceconfig['usemodules'] += ('_rawffi',)
+        spaceconfig['usemodules'] += ('_rawffi', '_cffi_backend')
     else:
         spaceconfig['usemodules'] += ('fcntl',)
 
diff --git a/pypy/module/_multiprocessing/test/test_win32.py b/pypy/module/_multiprocessing/test/test_win32.py
--- a/pypy/module/_multiprocessing/test/test_win32.py
+++ b/pypy/module/_multiprocessing/test/test_win32.py
@@ -2,7 +2,7 @@
 import sys
 
 class AppTestWin32:
-    spaceconfig = dict(usemodules=('_multiprocessing',
+    spaceconfig = dict(usemodules=('_multiprocessing', '_cffi_backend',
                                    'signal', '_rawffi', 'binascii'))
 
     def setup_class(cls):
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -30,6 +30,7 @@
 from pypy.objspace.std.unicodeobject import encode_object
 from pypy.module.__builtin__.descriptor import W_Property
 #from pypy.module.micronumpy.base import W_NDimArray
+from pypy.module.__pypy__.interp_buffer import W_Bufferable
 from rpython.rlib.entrypoint import entrypoint_lowlevel
 from rpython.rlib.rposix import FdValidator
 from rpython.rlib.unroll import unrolling_iterable
@@ -731,6 +732,7 @@
         'PyMethodDescr_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCMethodObject.typedef)',
         'PyWrapperDescr_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCWrapperObject.typedef)',
         'PyInstanceMethod_Type': 'space.gettypeobject(cpyext.classobject.InstanceMethod.typedef)',
+        'PyBufferable_Type': 'space.gettypeobject(W_Bufferable.typedef)',
         }.items():
         register_global(cpyname, 'PyTypeObject*', pypyexpr, header=pypy_decl)
 
@@ -1189,7 +1191,9 @@
     state.C.get_pyos_inputhook = rffi.llexternal(
         '_PyPy_get_PyOS_InputHook', [], FUNCPTR,
         compilation_info=eci, _nowrapper=True)
-
+    state.C.tuple_new = rffi.llexternal(
+        'tuple_new', [PyTypeObjectPtr, PyObject, PyObject], PyObject,
+        compilation_info=eci, _nowrapper=True)
 
 def init_function(func):
     INIT_FUNCTIONS.append(func)
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -32,8 +32,8 @@
  *     module/sys/version.py
  *     doc/conf.py
  */
-#define PYPY_VERSION "7.1.0-alpha0"
-#define PYPY_VERSION_NUM  0x07010000
+#define PYPY_VERSION "7.2.0-alpha0"
+#define PYPY_VERSION_NUM  0x07020000
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
    stays alive. */
diff --git a/pypy/module/cpyext/include/tupleobject.h b/pypy/module/cpyext/include/tupleobject.h
--- a/pypy/module/cpyext/include/tupleobject.h
+++ b/pypy/module/cpyext/include/tupleobject.h
@@ -18,6 +18,7 @@
 
 PyAPI_FUNC(PyObject *) PyTuple_New(Py_ssize_t size);
 PyAPI_FUNC(void) _PyPy_tuple_dealloc(PyObject *);
+PyAPI_FUNC(PyObject *) tuple_new(PyTypeObject *type, PyObject *args, PyObject *kwds);
 
 /* defined in varargswrapper.c */
 PyAPI_FUNC(PyObject *) PyTuple_Pack(Py_ssize_t, ...);
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -43,7 +43,9 @@
     fill_Py_buffer(space, w_obj.view, view)
     try:
         view.c_buf = rffi.cast(rffi.VOIDP, w_obj.view.get_raw_address())
-        view.c_obj = make_ref(space, w_userdata)
+        # not used in PyPy to keep something alive,
+        # but some c-extensions check the type without checking for NULL
+        view.c_obj = make_ref(space, space.w_None)
         rffi.setintfield(view, 'c_readonly', w_obj.view.readonly)
     except ValueError:
         w_s = w_obj.descr_tobytes(space)
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -46,15 +46,15 @@
     _dealloc(space, py_obj)
 
 def w_kwargs_from_args(space, __args__):
-    w_kwargs = None
-    if __args__.keywords:
-        # CCC: we should probably have a @jit.look_inside_iff if the
-        # keyword count is constant, as we do in Arguments.unpack
-        w_kwargs = space.newdict()
-        for i in range(len(__args__.keywords)):
-            key = __args__.keywords[i]
-            w_obj = __args__.keywords_w[i]
-            space.setitem(w_kwargs, space.newtext(key), w_obj)
+    if __args__.keywords is None:
+        return None
+    # CCC: we should probably have a @jit.look_inside_iff if the
+    # keyword count is constant, as we do in Arguments.unpack
+    w_kwargs = space.newdict()
+    for i in range(len(__args__.keywords)):
+        key = __args__.keywords[i]
+        w_obj = __args__.keywords_w[i]
+        space.setitem(w_kwargs, space.newtext(key), w_obj)
     return w_kwargs
 
 def undotted_name(name):
diff --git a/pypy/module/cpyext/parse/cpyext_memoryobject.h b/pypy/module/cpyext/parse/cpyext_memoryobject.h
--- a/pypy/module/cpyext/parse/cpyext_memoryobject.h
+++ b/pypy/module/cpyext/parse/cpyext_memoryobject.h
@@ -1,6 +1,12 @@
 /* The struct is declared here but it shouldn't
    be considered public. Don't access those fields directly,
    use the functions instead! */
+
+
+/* this is wrong, PyMemoryViewObject should use PyObject_VAR_HEAD, and use
+   ob_data[1] to hold the shapes, strides, and offsets for the view. Then
+   we should use specialized allocators (that break the cpyext model) to
+   allocate ob_data = malloc(sizeof(Py_ssize_t) * view.ndims * 3) */
 typedef struct {
     PyObject_HEAD
     Py_buffer view;
diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h
--- a/pypy/module/cpyext/parse/cpyext_object.h
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -52,7 +52,8 @@
 
 
 /* Py3k buffer interface, adapted for PyPy */
-#define Py_MAX_NDIMS 32
+/* XXX remove this constant, us a PyObject_VAR_HEAD instead */
+#define Py_MAX_NDIMS 36
 #define Py_MAX_FMT 128
 typedef struct bufferinfo {
     void *buf;
diff --git a/pypy/module/cpyext/src/tupleobject.c b/pypy/module/cpyext/src/tupleobject.c
--- a/pypy/module/cpyext/src/tupleobject.c
+++ b/pypy/module/cpyext/src/tupleobject.c
@@ -89,3 +89,48 @@
 done:
     Py_TRASHCAN_SAFE_END(op)
 }
+
+static PyObject *
+tuple_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds);
+
+PyObject *
+tuple_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+{
+    PyObject *arg = NULL;
+    static char *kwlist[] = {"sequence", 0};
+
+    if (type != &PyTuple_Type)
+        return tuple_subtype_new(type, args, kwds);
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O:tuple", kwlist, &arg))
+        return NULL;
+
+    if (arg == NULL)
+        return PyTuple_New(0);
+    else
+        return PySequence_Tuple(arg);
+}
+
+static PyObject *
+tuple_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+{
+    PyObject *tmp, *newobj, *item;
+    Py_ssize_t i, n;
+
+    assert(PyType_IsSubtype(type, &PyTuple_Type));
+    tmp = tuple_new(&PyTuple_Type, args, kwds);
+    if (tmp == NULL)
+        return NULL;
+    assert(PyTuple_Check(tmp));
+    newobj = type->tp_alloc(type, n = PyTuple_GET_SIZE(tmp));
+    if (newobj == NULL)
+        return NULL;
+    for (i = 0; i < n; i++) {
+        item = PyTuple_GET_ITEM(tmp, i);
+        Py_INCREF(item);
+        PyTuple_SET_ITEM(newobj, i, item);
+    }
+    Py_DECREF(tmp);
+    return newobj;
+}
+
+
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -36,6 +36,23 @@
         decref(space, ref)
         decref(space, c_memoryview)
 
+    def test_class_with___buffer__(self, space, api):
+        w_obj = space.appexec([], """():
+            from __pypy__.bufferable import bufferable
+            class B(bufferable):
+                def __init__(self):
+                    self.buf = bytearray(10)
+
+                def __buffer__(self, flags):
+                    return memoryview(self.buf)
+            return B()""")
+        py_obj = make_ref(space, w_obj)
+        assert py_obj.c_ob_type.c_tp_as_buffer
+        assert py_obj.c_ob_type.c_tp_as_buffer.c_bf_getbuffer
+        assert py_obj.c_ob_type.c_tp_as_buffer.c_bf_getreadbuffer
+        assert py_obj.c_ob_type.c_tp_as_buffer.c_bf_getwritebuffer
+         
+
 class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase):
     def test_fillWithObject(self):
         module = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/test/test_methodobject.py b/pypy/module/cpyext/test/test_methodobject.py
--- a/pypy/module/cpyext/test/test_methodobject.py
+++ b/pypy/module/cpyext/test/test_methodobject.py
@@ -87,6 +87,7 @@
         assert mod.getarg_KW(a=3, b=4) == ((), {'a': 3, 'b': 4})
         assert mod.getarg_KW(1, 2, a=3, b=4) == ((1, 2), {'a': 3, 'b': 4})
         assert mod.getarg_KW.__name__ == "getarg_KW"
+        assert mod.getarg_KW(*(), **{}) == ((), {})
 
 
     def test_func_attributes(self):
diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py
--- a/pypy/module/cpyext/test/test_tupleobject.py
+++ b/pypy/module/cpyext/test/test_tupleobject.py
@@ -226,3 +226,44 @@
             raises(SystemError, module.set_after_use, s)
         else:
             module.set_after_use(s)
+
+    def test_mp_length(self):
+        # issue 2968: creating a subclass of tuple in C led to recursion
+        # since the default tp_new needs to build a w_obj, but that needs
+        # to call space.len_w, which needs to call tp_new.
+        module = self.import_extension('foo', [
+            ("get_size", "METH_NOARGS",
+             """
+                return (PyObject*)&THPSizeType;
+             """),
+            ], prologue='''
+                #include "Python.h"
+
+                struct THPSize {
+                  PyTupleObject tuple;
+                } THPSize;
+
+                static PyMappingMethods THPSize_as_mapping = {
+                    0, //PyTuple_Type.tp_as_mapping->mp_length,
+                    0,
+                    0
+                };
+
+                PyTypeObject THPSizeType = {
+                  PyVarObject_HEAD_INIT(0, 0)
+                  "torch.Size",                          /* tp_name */
+                  sizeof(THPSize),                       /* tp_basicsize */
+                };
+            ''' , more_init = '''
+                THPSize_as_mapping.mp_length = PyTuple_Type.tp_as_mapping->mp_length;
+                THPSizeType.tp_base = &PyTuple_Type;
+                THPSizeType.tp_flags = Py_TPFLAGS_DEFAULT;
+                THPSizeType.tp_as_mapping = &THPSize_as_mapping;
+                THPSizeType.tp_new = PyTuple_Type.tp_new;
+                if (PyType_Ready(&THPSizeType) < 0) INITERROR;
+            ''')
+        SZ = module.get_size()
+        s = SZ((1, 2, 3))
+        assert len(s) == 3
+        assert len(s) == 3
+
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -686,6 +686,11 @@
         update_all_slots(space, w_type, pto)
     else:
         update_all_slots_builtin(space, w_type, pto)
+
+    # XXX generlize this pattern for various slot functions implemented in C
+    if space.is_w(w_type, space.w_tuple):
+        pto.c_tp_new = state.C.tuple_new
+
     if not pto.c_tp_new:
         base_object_pyo = make_ref(space, space.w_object)
         base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo)
diff --git a/pypy/module/errno/interp_errno.py b/pypy/module/errno/interp_errno.py
--- a/pypy/module/errno/interp_errno.py
+++ b/pypy/module/errno/interp_errno.py
@@ -43,6 +43,18 @@
     "WSAGETASYNCBUFLE", "WSAEDESTADDRREQ", "WSAECONNREFUSED", "WSAENETRESET",
     "WSAN", "WSAEDQUOT"]
 
+# The following constants were added to errno.h in VS2010 but have
+# preferred WSA equivalents, so errno.EADDRINUSE == errno.WSAEADDRINUSE.
+win_errors_override = [
+    "WSAEADDRINUSE", "WSAEADDRNOTAVAI", "WSAEAFNOSUPPORT", "WSAEALREADY",
+    "WSAECONNABORTED", "WSAECONNREFUSED", "WSAECONNRESET", "WSAEDESTADDRREQ",
+    "WSAEHOSTUNREACH", "WSAEINPROGRESS", "WSAEISCONN", "WSAELOOP",
+    "WSAEMSGSIZE", "WSAENETDOWN", "WSAENETRESET", "WSAENETUNREACH",
+    "WSAENOBUFS", "WSAENOPROTOOPT", "WSAENOTCONN", "WSAENOTSOCK",
+    "WSAEOPNOTSUPP", "WSAEPROTONOSUPPORT", "WSAEPROTOTYPE", "WSAETIMEDOUT",
+    "WSAEWOULDBLOCK",
+    ]
+
 more_errors = [
     "ENOMEDIUM", "EMEDIUMTYPE", "ECANCELED", "ENOKEY", "EKEYEXPIRED",
     "EKEYREVOKED", "EKEYREJECTED", "EOWNERDEAD", "ENOTRECOVERABLE", "ERFKILL",
@@ -80,7 +92,8 @@
     assert name.startswith('WSA')
     code = config[name]
     if code is not None:
-        if name[3:] in errors and name[3:] not in name2code:
+        if name[3:] in errors and (name in win_errors_override or 
+                                   name[3:] not in name2code):
             # errno.EFOO = 
             name2code[name[3:]] = code
         # errno.WSABAR = 
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -13,7 +13,7 @@
 # make sure to keep PYPY_VERSION in sync with:
 #    module/cpyext/include/patchlevel.h
 #    doc/conf.py
-PYPY_VERSION               = (7, 1, 0, "alpha", 0)
+PYPY_VERSION               = (7, 2, 0, "alpha", 0)
 
 
 import pypy
diff --git a/pypy/module/zlib/interp_zlib.py b/pypy/module/zlib/interp_zlib.py
--- a/pypy/module/zlib/interp_zlib.py
+++ b/pypy/module/zlib/interp_zlib.py
@@ -313,6 +313,11 @@
         try:
             self.lock()
             try:
+                if not self.stream:
+                    raise oefmt(
+                        space.w_ValueError,
+                        "Decompressor was already flushed",
+                    )
                 copied = rzlib.inflateCopy(self.stream)
             finally:
                 self.unlock()
@@ -338,6 +343,9 @@
             if length <= 0:
                 raise oefmt(space.w_ValueError,
                             "length must be greater than zero")
+        if not self.stream:
+            raise zlib_error(space,
+                             "compressor object already flushed")
         data = self.unconsumed_tail
         try:
             self.lock()

From pypy.commits at gmail.com  Wed Mar 27 03:35:26 2019
From: pypy.commits at gmail.com (andrewjlawrence)
Date: Wed, 27 Mar 2019 00:35:26 -0700 (PDT)
Subject: [pypy-commit] pypy winoverlapped: Modify exceptions to produce
 connection refused error on client socket failure.
Message-ID: <5c9b27be.1c69fb81.4b7bd.dad8@mx.google.com>

Author: andrewjlawrence
Branch: winoverlapped
Changeset: r96354:e824a633d633
Date: 2019-03-19 06:04 +0000
http://bitbucket.org/pypy/pypy/changeset/e824a633d633/

Log:	Modify exceptions to produce connection refused error on client
	socket failure.

diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -261,7 +261,7 @@
                 132: 13, 145: 41, 158: 13, 161: 2, 164: 11, 167: 13, 183: 17,
                 188: 8, 189: 8, 190: 8, 191: 8, 192: 8, 193: 8, 194: 8,
                 195: 8, 196: 8, 197: 8, 198: 8, 199: 8, 200: 8, 201: 8,
-                202: 8, 206: 2, 215: 11, 232: 32, 267: 20, 1816: 12,
+                202: 8, 206: 2, 215: 11, 232: 32, 267: 20, 1816: 12, 1225: 111,
                 }
         return errors, errno.EINVAL
 

From pypy.commits at gmail.com  Wed Mar 27 03:35:28 2019
From: pypy.commits at gmail.com (andrewjlawrence)
Date: Wed, 27 Mar 2019 00:35:28 -0700 (PDT)
Subject: [pypy-commit] pypy winoverlapped: Fixed a few more bugs realted to
 overlapped io in test_events.py
Message-ID: <5c9b27c0.1c69fb81.7d69c.2831@mx.google.com>

Author: andrewjlawrence
Branch: winoverlapped
Changeset: r96355:9042d6247856
Date: 2019-03-27 07:26 +0000
http://bitbucket.org/pypy/pypy/changeset/9042d6247856/

Log:	Fixed a few more bugs realted to overlapped io in test_events.py

diff --git a/lib_pypy/_overlapped.py b/lib_pypy/_overlapped.py
--- a/lib_pypy/_overlapped.py
+++ b/lib_pypy/_overlapped.py
@@ -206,7 +206,7 @@
             self.error = err
 
         if err != _winapi.ERROR_SUCCESS and err != _winapi.ERROR_MORE_DATA:
-            if not (err == _winapi.ERROR_BROKEN_PIPE and (self.type == TYPE_READ or self.type == TYPE_READINTO)):
+            if not (err == _winapi.ERROR_BROKEN_PIPE and (self.type in [OverlappedType.TYPE_READ, OverlappedType.TYPE_READINTO])):
                 raise _winapi._WinError()
 
         if self.type == OverlappedType.TYPE_READ:
@@ -502,8 +502,6 @@
     pdata = _ffi.cast("PostCallbackData *", lpparameter)
     ret = _kernel32.PostQueuedCompletionStatus(pdata.hCompletionPort, timerorwaitfired, _ffi.cast("ULONG_PTR",0), pdata.Overlapped)
     result = False
-#    if not ret:
-#         err = _winapi._WinError()
 
 
 def RegisterWaitWithQueue(object, completionport, ovaddress, miliseconds):
@@ -512,7 +510,7 @@
     data[0].hCompletionPort = completionport
     data[0].Overlapped = ovaddress
     ret = _kernel32.RegisterWaitForSingleObject(newwaitobject,
-                                                object,
+                                                _int2handle(object),
                                                 _ffi.cast("WAITORTIMERCALLBACK",post_to_queue_callback),
                                                 data,
                                                 miliseconds, 
diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py
--- a/lib_pypy/_winapi.py
+++ b/lib_pypy/_winapi.py
@@ -52,7 +52,7 @@
     handle = _kernel32.CreateFileW(*args)
     if handle == INVALID_HANDLE_VALUE:
         raise _WinError()
-    return handle
+    return _handle2int(handle)
 
 def SetNamedPipeHandleState(namedpipe, mode, max_collection_count, collect_data_timeout):
     d0 = _ffi.new('DWORD[1]', [mode])

From pypy.commits at gmail.com  Wed Mar 27 03:39:39 2019
From: pypy.commits at gmail.com (mattip)
Date: Wed, 27 Mar 2019 00:39:39 -0700 (PDT)
Subject: [pypy-commit] pypy winoverlapped: merge heads
Message-ID: <5c9b28bb.1c69fb81.f217f.ec16@mx.google.com>

Author: Matti Picus 
Branch: winoverlapped
Changeset: r96356:01451c1ab8c5
Date: 2019-03-27 09:37 +0200
http://bitbucket.org/pypy/pypy/changeset/01451c1ab8c5/

Log:	merge heads

diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -261,7 +261,7 @@
                 132: 13, 145: 41, 158: 13, 161: 2, 164: 11, 167: 13, 183: 17,
                 188: 8, 189: 8, 190: 8, 191: 8, 192: 8, 193: 8, 194: 8,
                 195: 8, 196: 8, 197: 8, 198: 8, 199: 8, 200: 8, 201: 8,
-                202: 8, 206: 2, 215: 11, 232: 32, 267: 20, 1816: 12,
+                202: 8, 206: 2, 215: 11, 232: 32, 267: 20, 1816: 12, 1225: 111,
                 }
         return errors, errno.EINVAL
 

From pypy.commits at gmail.com  Wed Mar 27 03:39:41 2019
From: pypy.commits at gmail.com (andrewjlawrence)
Date: Wed, 27 Mar 2019 00:39:41 -0700 (PDT)
Subject: [pypy-commit] pypy default: Modify exceptions to produce connection
 refused error on client socket failure.
Message-ID: <5c9b28bd.1c69fb81.6dc2c.cb08@mx.google.com>

Author: andrewjlawrence
Branch: 
Changeset: r96357:43cdaf1865f0
Date: 2019-03-19 06:04 +0000
http://bitbucket.org/pypy/pypy/changeset/43cdaf1865f0/

Log:	Modify exceptions to produce connection refused error on client
	socket failure.

diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -261,7 +261,7 @@
                 132: 13, 145: 41, 158: 13, 161: 2, 164: 11, 167: 13, 183: 17,
                 188: 8, 189: 8, 190: 8, 191: 8, 192: 8, 193: 8, 194: 8,
                 195: 8, 196: 8, 197: 8, 198: 8, 199: 8, 200: 8, 201: 8,
-                202: 8, 206: 2, 215: 11, 232: 32, 267: 20, 1816: 12,
+                202: 8, 206: 2, 215: 11, 232: 32, 267: 20, 1816: 12, 1225: 111,
                 }
         return errors, errno.EINVAL
 

From pypy.commits at gmail.com  Wed Mar 27 03:39:43 2019
From: pypy.commits at gmail.com (mattip)
Date: Wed, 27 Mar 2019 00:39:43 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge default into branch
Message-ID: <5c9b28bf.1c69fb81.b60f.bb1f@mx.google.com>

Author: Matti Picus 
Branch: py3.6
Changeset: r96358:f774aae69691
Date: 2019-03-27 09:38 +0200
http://bitbucket.org/pypy/pypy/changeset/f774aae69691/

Log:	merge default into branch

diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -261,7 +261,7 @@
                 132: 13, 145: 41, 158: 13, 161: 2, 164: 11, 167: 13, 183: 17,
                 188: 8, 189: 8, 190: 8, 191: 8, 192: 8, 193: 8, 194: 8,
                 195: 8, 196: 8, 197: 8, 198: 8, 199: 8, 200: 8, 201: 8,
-                202: 8, 206: 2, 215: 11, 232: 32, 267: 20, 1816: 12,
+                202: 8, 206: 2, 215: 11, 232: 32, 267: 20, 1816: 12, 1225: 111,
                 }
         return errors, errno.EINVAL
 

From pypy.commits at gmail.com  Wed Mar 27 06:45:19 2019
From: pypy.commits at gmail.com (guil...@Guillaumes-MacBook-Pro.local)
Date: Wed, 27 Mar 2019 03:45:19 -0700 (PDT)
Subject: [pypy-commit] cffi default: fix #407 add support for u/U suffix in
 integer constants (eg. 0xABu, or 0xCDU).
Message-ID: <5c9b543f.1c69fb81.a0ffe.0215@mx.google.com>

Author: guillaumesottas at Guillaumes-MacBook-Pro.local
Branch: 
Changeset: r3248:c0fbaf8dc825
Date: 2019-03-25 10:24 -0600
http://bitbucket.org/cffi/cffi/changeset/c0fbaf8dc825/

Log:	fix #407 add support for u/U suffix in integer constants (eg. 0xABu,
	or 0xCDU).

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -819,6 +819,8 @@
             s = exprnode.value
             if s.startswith('0'):
                 if s.startswith('0x') or s.startswith('0X'):
+                    if s.endswith('u') or s.endswith('U'):
+                        s = s[:-1]
                     return int(s, 16)
                 return int(s, 8)
             elif '1' <= s[0] <= '9':
diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py
--- a/testing/cffi0/test_parsing.py
+++ b/testing/cffi0/test_parsing.py
@@ -466,3 +466,10 @@
     e = py.test.raises(CDefError, ffi.cdef, 'void foo(void) {}')
     assert str(e.value) == (':1: unexpected : '
                             'this construct is valid C but not valid in cdef()')
+
+def test_unsigned_int_suffix_for_constant():
+    ffi = FFI()
+    ffi.cdef("""enum e {
+                    enumerator_0=0x00,
+                    enumerator_1=0x01u,
+                    enumerator_1=0x01U};""")

From pypy.commits at gmail.com  Wed Mar 27 06:45:20 2019
From: pypy.commits at gmail.com (guil...@Guillaumes-MacBook-Pro.local)
Date: Wed, 27 Mar 2019 03:45:20 -0700 (PDT)
Subject: [pypy-commit] cffi default: add support for long/long long C
 integer constant suffixes, and support
Message-ID: <5c9b5440.1c69fb81.6b988.eb0b@mx.google.com>

Author: guillaumesottas at Guillaumes-MacBook-Pro.local
Branch: 
Changeset: r3249:d1bf39c55881
Date: 2019-03-26 13:09 -0600
http://bitbucket.org/cffi/cffi/changeset/d1bf39c55881/

Log:	add support for long/long long C integer constant suffixes, and
	support for base 2 integer constant as well.

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -817,12 +817,22 @@
         # or positive/negative number
         if isinstance(exprnode, pycparser.c_ast.Constant):
             s = exprnode.value
-            if s.startswith('0'):
-                if s.startswith('0x') or s.startswith('0X'):
-                    if s.endswith('u') or s.endswith('U'):
-                        s = s[:-1]
-                    return int(s, 16)
-                return int(s, 8)
+            if '0' <= s[0] <= '9':
+                s = s.rstrip('uUlL')  # remove integer constant suffix if any. this will remove pattern such as lL, but
+                # it is the responsibility of the C parser to perform this check.
+                try:  # first we try to convert to base 8/10, as it will fail if the string contains base 2/16 C prefix.
+                    if s.startswith('0'):
+                        return int(s, 8)
+                    else:
+                        return int(s, 10)
+                except ValueError:  # then it should be a base 2 or a base 16. it is necessary to explicitly check the
+                    # prefix, as python's int() function will be able to convert both (0b01 and 0x0b01) into base 16.
+                    if len(s) > 1:
+                        if s.lower()[0:2] == '0x':
+                            return int(s, 16)
+                        elif s.lower()[0:2] == '0b':
+                            return int(s, 2)
+                raise CDefError("invalid constant %r" % (s,))
             elif '1' <= s[0] <= '9':
                 return int(s, 10)
             elif s[0] == "'" and s[-1] == "'" and (
diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py
--- a/testing/cffi0/test_parsing.py
+++ b/testing/cffi0/test_parsing.py
@@ -470,6 +470,39 @@
 def test_unsigned_int_suffix_for_constant():
     ffi = FFI()
     ffi.cdef("""enum e {
-                    enumerator_0=0x00,
-                    enumerator_1=0x01u,
-                    enumerator_1=0x01U};""")
+                    bin_0=0b10,
+                    bin_1=0b10u,
+                    bin_2=0b10U,
+                    bin_3=0b10l,
+                    bin_4=0b10L,
+                    bin_5=0b10ll,
+                    bin_6=0b10LL,
+                    oct_0=010,
+                    oct_1=010u,
+                    oct_2=010U,
+                    oct_3=010l,
+                    oct_4=010L,
+                    oct_5=010ll,
+                    oct_6=010LL,
+                    dec_0=10,
+                    dec_1=10u,
+                    dec_2=10U,
+                    dec_3=10l,
+                    dec_4=10L,
+                    dec_5=10ll,
+                    dec_6=10LL,
+                    hex_0=0x10,
+                    hex_1=0x10u,
+                    hex_2=0x10U,
+                    hex_3=0x10l,
+                    hex_4=0x10L,
+                    hex_5=0x10ll,
+                    hex_6=0x10LL,};""")
+    needs_dlopen_none()
+    C = ffi.dlopen(None)
+    for base, expected_result in (('bin', 2), ('oct', 8), ('dec', 10), ('hex', 16)):
+        for index in range(7):
+            try:
+                assert getattr(C, '{base}_{index}'.format(base=base, index=index)) == expected_result
+            except AssertionError as e:
+                raise e

From pypy.commits at gmail.com  Wed Mar 27 06:45:22 2019
From: pypy.commits at gmail.com (guil...@Guillaumes-MacBook-Pro.local)
Date: Wed, 27 Mar 2019 03:45:22 -0700 (PDT)
Subject: [pypy-commit] cffi default: remove useless try/catch,
 and remove unreachable elif statement.
Message-ID: <5c9b5442.1c69fb81.a0ffe.021c@mx.google.com>

Author: guillaumesottas at Guillaumes-MacBook-Pro.local
Branch: 
Changeset: r3250:ae7712fab365
Date: 2019-03-26 17:12 -0600
http://bitbucket.org/cffi/cffi/changeset/ae7712fab365/

Log:	remove useless try/catch, and remove unreachable elif statement.

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -833,8 +833,6 @@
                         elif s.lower()[0:2] == '0b':
                             return int(s, 2)
                 raise CDefError("invalid constant %r" % (s,))
-            elif '1' <= s[0] <= '9':
-                return int(s, 10)
             elif s[0] == "'" and s[-1] == "'" and (
                     len(s) == 3 or (len(s) == 4 and s[1] == "\\")):
                 return ord(s[-2])
diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py
--- a/testing/cffi0/test_parsing.py
+++ b/testing/cffi0/test_parsing.py
@@ -502,7 +502,4 @@
     C = ffi.dlopen(None)
     for base, expected_result in (('bin', 2), ('oct', 8), ('dec', 10), ('hex', 16)):
         for index in range(7):
-            try:
-                assert getattr(C, '{base}_{index}'.format(base=base, index=index)) == expected_result
-            except AssertionError as e:
-                raise e
+            assert getattr(C, '{base}_{index}'.format(base=base, index=index)) == expected_result

From pypy.commits at gmail.com  Wed Mar 27 06:45:24 2019
From: pypy.commits at gmail.com (guil...@Guillaumes-MacBook-Pro.local)
Date: Wed, 27 Mar 2019 03:45:24 -0700 (PDT)
Subject: [pypy-commit] cffi default: remove extra comments.
Message-ID: <5c9b5444.1c69fb81.84b7f.1385@mx.google.com>

Author: guillaumesottas at Guillaumes-MacBook-Pro.local
Branch: 
Changeset: r3251:fa1ef05714de
Date: 2019-03-26 17:23 -0600
http://bitbucket.org/cffi/cffi/changeset/fa1ef05714de/

Log:	remove extra comments.

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -818,15 +818,13 @@
         if isinstance(exprnode, pycparser.c_ast.Constant):
             s = exprnode.value
             if '0' <= s[0] <= '9':
-                s = s.rstrip('uUlL')  # remove integer constant suffix if any. this will remove pattern such as lL, but
-                # it is the responsibility of the C parser to perform this check.
-                try:  # first we try to convert to base 8/10, as it will fail if the string contains base 2/16 C prefix.
+                s = s.rstrip('uUlL')
+                try:
                     if s.startswith('0'):
                         return int(s, 8)
                     else:
                         return int(s, 10)
-                except ValueError:  # then it should be a base 2 or a base 16. it is necessary to explicitly check the
-                    # prefix, as python's int() function will be able to convert both (0b01 and 0x0b01) into base 16.
+                except ValueError:
                     if len(s) > 1:
                         if s.lower()[0:2] == '0x':
                             return int(s, 16)

From pypy.commits at gmail.com  Wed Mar 27 07:26:34 2019
From: pypy.commits at gmail.com (arigo)
Date: Wed, 27 Mar 2019 04:26:34 -0700 (PDT)
Subject: [pypy-commit] pypy default: Unicode characters out of
 range(0x11000): fix a few docstrings, and try to more
Message-ID: <5c9b5dea.1c69fb81.89104.4966@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r96359:a59c3b47eec9
Date: 2019-03-27 12:26 +0100
http://bitbucket.org/pypy/pypy/changeset/a59c3b47eec9/

Log:	Unicode characters out of range(0x11000): fix a few docstrings, and
	try to more systematically test (and fix) various corner cases

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -537,14 +537,17 @@
 def wcharpsize2utf8(space, wcharp, size):
     """Safe version of rffi.wcharpsize2utf8.
 
-    Raises app-level rutf8.OutOfRange if any wchar value is outside the valid
+    Raises app-level ValueError if any wchar value is outside the valid
     codepoint range.
     """
     try:
         return rffi.wcharpsize2utf8(wcharp, size)
     except rutf8.OutOfRange as e:
-        raise oefmt(space.w_ValueError,
-            "character %s is not in range [U+0000; U+10ffff]", 'U+%x' % e.code)
+        raise wrap_unicode_out_of_range_error(space, e)
+
+def wrap_unicode_out_of_range_error(space, e):
+    raise oefmt(space.w_ValueError,
+        "character %s is not in range [U+0000; U+10ffff]", 'U+%x' % e.code)
 
 
 # ____________________________________________________________
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -5,6 +5,8 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import interp_attrproperty
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
+from pypy.interpreter.unicodehelper import wcharpsize2utf8
+from pypy.interpreter.unicodehelper import wrap_unicode_out_of_range_error
 
 from rpython.rlib.clibffi import *
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -596,10 +598,13 @@
     if address == 0:
         return space.w_None
     wcharp_addr = rffi.cast(rffi.CWCHARP, address)
-    if maxlength == -1:
-        s, lgt = rffi.wcharp2utf8(wcharp_addr)
-    else:
-        s, lgt = rffi.wcharp2utf8n(wcharp_addr, maxlength)
+    try:
+        if maxlength == -1:
+            s, lgt = rffi.wcharp2utf8(wcharp_addr)
+        else:
+            s, lgt = rffi.wcharp2utf8n(wcharp_addr, maxlength)
+    except rutf8.OutOfRange as e:
+        raise wrap_unicode_out_of_range_error(space, e)
     return space.newutf8(s, lgt)
 
 @unwrap_spec(address=r_uint, maxlength=int)
@@ -613,7 +618,7 @@
 def wcharp2rawunicode(space, address, maxlength=-1):
     if maxlength == -1:
         return wcharp2unicode(space, address)
-    s = rffi.wcharpsize2utf8(rffi.cast(rffi.CWCHARP, address), maxlength)
+    s = wcharpsize2utf8(space, rffi.cast(rffi.CWCHARP, address), maxlength)
     return space.newutf8(s, maxlength)
 
 @unwrap_spec(address=r_uint, newcontent='bufferstr')
diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py
--- a/pypy/module/_rawffi/test/test__rawffi.py
+++ b/pypy/module/_rawffi/test/test__rawffi.py
@@ -1229,6 +1229,23 @@
         lib = _rawffi.CDLL(self.lib_name)
         assert lib.name == self.lib_name
 
+    def test_wcharp2rawunicode(self):
+        import _rawffi
+        A = _rawffi.Array('i')
+        arg = A(1)
+        arg[0] = 0x1234
+        u = _rawffi.wcharp2rawunicode(arg.itemaddress(0))
+        assert u == u'\u1234'
+        u = _rawffi.wcharp2rawunicode(arg.itemaddress(0), 1)
+        assert u == u'\u1234'
+        arg[0] = -1
+        raises(ValueError, _rawffi.wcharp2rawunicode, arg.itemaddress(0))
+        raises(ValueError, _rawffi.wcharp2rawunicode, arg.itemaddress(0), 1)
+        arg[0] = 0x110000
+        raises(ValueError, _rawffi.wcharp2rawunicode, arg.itemaddress(0))
+        raises(ValueError, _rawffi.wcharp2rawunicode, arg.itemaddress(0), 1)
+        arg.free()
+
 
 class AppTestAutoFree:
     spaceconfig = dict(usemodules=['_rawffi', 'struct'])
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -13,6 +13,7 @@
     interp2app, interpindirect2app, unwrap_spec)
 from pypy.interpreter.typedef import (
     GetSetProperty, TypeDef, make_weakref_descr)
+from pypy.interpreter.unicodehelper import wcharpsize2utf8
 from pypy.module._file.interp_file import W_File
 
 
@@ -463,7 +464,8 @@
         """
         if self.typecode == 'u':
             buf = rffi.cast(UNICODE_ARRAY, self._buffer_as_unsigned())
-            return space.newutf8(rffi.wcharpsize2utf8(buf, self.len), self.len)
+            utf8 = wcharpsize2utf8(space, buf, self.len)
+            return space.newutf8(utf8, self.len)
         else:
             raise oefmt(space.w_ValueError,
                         "tounicode() may only be called on type 'u' arrays")
@@ -714,8 +716,15 @@
             s = "array('%s', %s)" % (self.typecode, space.text_w(r))
             return space.newtext(s)
         elif self.typecode == "u":
-            r = space.repr(self.descr_tounicode(space))
-            s = "array('%s', %s)" % (self.typecode, space.text_w(r))
+            try:
+                w_unicode = self.descr_tounicode(space)
+            except OperationError as e:
+                if not e.match(space, space.w_ValueError):
+                    raise
+                r = ''
+            else:
+                r = space.text_w(space.repr(w_unicode))
+            s = "array('%s', %s)" % (self.typecode, r)
             return space.newtext(s)
         else:
             r = space.repr(self.descr_tolist(space))
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -847,10 +847,15 @@
         assert repr(mya('i', (1, 2, 3))) == "array('i', [1, 2, 3])"
 
     def test_unicode_outofrange(self):
-        a = self.array('u', u'\x01\u263a\x00\ufeff')
-        b = self.array('u', u'\x01\u263a\x00\ufeff')
+        input_unicode = u'\x01\u263a\x00\ufeff'
+        a = self.array('u', input_unicode)
+        b = self.array('u', input_unicode)
         b.byteswap()
         assert a != b
+        assert str(a) == "array('u', %r)" % (input_unicode,)
+        assert str(b) == "array('u', )"
+        assert a.tounicode() == input_unicode
+        raises(ValueError, b.tounicode)   # doesn't work
 
     def test_weakref(self):
         import weakref
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -1023,7 +1023,7 @@
 def wcharpsize2utf8(w, size):
     """ Helper to convert WCHARP pointer to utf8 in one go.
     Equivalent to wcharpsize2unicode().encode("utf8")
-    Raises ValueError if characters are outside range(0x110000)!
+    Raises rutf8.OutOfRange if characters are outside range(0x110000)!
     """
     from rpython.rlib import rutf8
 
@@ -1033,6 +1033,9 @@
     return s.build()
 
 def wcharp2utf8(w):
+    """
+    Raises rutf8.OutOfRange if characters are outside range(0x110000)!
+    """
     from rpython.rlib import rutf8
 
     s = rutf8.Utf8StringBuilder()
@@ -1043,6 +1046,9 @@
     return s.build(), i
 
 def wcharp2utf8n(w, maxlen):
+    """
+    Raises rutf8.OutOfRange if characters are outside range(0x110000)!
+    """
     from rpython.rlib import rutf8
 
     s = rutf8.Utf8StringBuilder(maxlen)

From pypy.commits at gmail.com  Wed Mar 27 07:37:21 2019
From: pypy.commits at gmail.com (arigo)
Date: Wed, 27 Mar 2019 04:37:21 -0700 (PDT)
Subject: [pypy-commit] pypy default: Be more informative, just because we can
Message-ID: <5c9b6071.1c69fb81.75737.38e5@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r96360:fa16b2515a57
Date: 2019-03-27 12:36 +0100
http://bitbucket.org/pypy/pypy/changeset/fa16b2515a57/

Log:	Be more informative, just because we can

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -721,7 +721,8 @@
             except OperationError as e:
                 if not e.match(space, space.w_ValueError):
                     raise
-                r = ''
+                w_exc_value = e.get_w_value(space)
+                r = "<%s>" % (space.str_w(w_exc_value),)
             else:
                 r = space.text_w(space.repr(w_unicode))
             s = "array('%s', %s)" % (self.typecode, r)
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -853,7 +853,8 @@
         b.byteswap()
         assert a != b
         assert str(a) == "array('u', %r)" % (input_unicode,)
-        assert str(b) == "array('u', )"
+        assert str(b) == ("array('u', )")
         assert a.tounicode() == input_unicode
         raises(ValueError, b.tounicode)   # doesn't work
 

From pypy.commits at gmail.com  Wed Mar 27 08:15:51 2019
From: pypy.commits at gmail.com (arigo)
Date: Wed, 27 Mar 2019 05:15:51 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix the "not we_are_translated()" path,
 almost never used
Message-ID: <5c9b6977.1c69fb81.63a4.eaf3@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r96361:8fba932de88a
Date: 2019-03-27 13:15 +0100
http://bitbucket.org/pypy/pypy/changeset/8fba932de88a/

Log:	Fix the "not we_are_translated()" path, almost never used

diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py
--- a/pypy/module/gc/referents.py
+++ b/pypy/module/gc/referents.py
@@ -20,7 +20,7 @@
     # inherits from W_Root for internal reasons.  Such instances don't
     # have a typedef at all (or have a null typedef after translation).
     if not we_are_translated():
-        if not hasattr(w_obj, 'typedef'):
+        if getattr(w_obj, 'typedef', None) is None:
             return None
     else:
         if w_obj is None or not w_obj.typedef:

From pypy.commits at gmail.com  Wed Mar 27 10:39:51 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 27 Mar 2019 07:39:51 -0700 (PDT)
Subject: [pypy-commit] pypy default: issue2980: stop using arbitrarily much
 stack for building constant lists and
Message-ID: <5c9b8b37.1c69fb81.f7144.1a00@mx.google.com>

Author: Carl Friedrich Bolz-Tereick 
Branch: 
Changeset: r96362:c46e92cbc2f9
Date: 2019-03-27 14:41 +0100
http://bitbucket.org/pypy/pypy/changeset/c46e92cbc2f9/

Log:	issue2980: stop using arbitrarily much stack for building constant
	lists and sets. do it element by element instead. otherwise the JIT
	cannot produce any code for any of the loops in the same scope,
	because it runs into limitations.

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -19,6 +19,7 @@
     symbols = symtable.SymtableBuilder(space, module, info)
     return TopLevelCodeGenerator(space, module, symbols, info).assemble()
 
+MAX_STACKDEPTH_CONTAINERS = 100
 
 name_ops_default = misc.dict_to_switch({
     ast.Load: ops.LOAD_NAME,
@@ -920,9 +921,17 @@
         elt_count = len(l.elts) if l.elts is not None else 0
         if l.ctx == ast.Store:
             self.emit_op_arg(ops.UNPACK_SEQUENCE, elt_count)
-        self.visit_sequence(l.elts)
-        if l.ctx == ast.Load:
-            self.emit_op_arg(ops.BUILD_LIST, elt_count)
+        if elt_count > MAX_STACKDEPTH_CONTAINERS:
+            # pushing all the elements would make the stack depth gigantic.
+            # build the list incrementally instead
+            self.emit_op_arg(ops.BUILD_LIST, 0)
+            for element in l.elts:
+                element.walkabout(self)
+                self.emit_op_arg(ops.LIST_APPEND, 1)
+        else:
+            self.visit_sequence(l.elts)
+            if l.ctx == ast.Load:
+                self.emit_op_arg(ops.BUILD_LIST, elt_count)
 
     def visit_Dict(self, d):
         self.update_position(d.lineno)
@@ -936,8 +945,15 @@
     def visit_Set(self, s):
         self.update_position(s.lineno)
         elt_count = len(s.elts) if s.elts is not None else 0
-        self.visit_sequence(s.elts)
-        self.emit_op_arg(ops.BUILD_SET, elt_count)
+        if elt_count > MAX_STACKDEPTH_CONTAINERS:
+            self.emit_op_arg(ops.BUILD_SET, 0)
+            for element in s.elts:
+                element.walkabout(self)
+                self.emit_op_arg(ops.SET_ADD, 1)
+        else:
+            self.visit_sequence(s.elts)
+            self.emit_op_arg(ops.BUILD_SET, elt_count)
+
 
     def visit_Name(self, name):
         self.update_position(name.lineno)
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1218,3 +1218,24 @@
         counts = self.count_instructions(source)
         assert ops.BUILD_TUPLE not in counts
 
+
+class TestHugeStackDepths:
+    def test_list(self):
+        space = self.space
+        source = "a = [" + ",".join([str(i) for i in range(200)]) + "]\n"
+        code = compile_with_astcompiler(source, 'exec', space)
+        assert code.co_stacksize < 100
+        w_dict = space.newdict()
+        code.exec_code(space, w_dict, w_dict)
+        assert space.unwrap(space.getitem(w_dict, space.newtext("a"))) == range(200)
+
+    def test_set(self):
+        space = self.space
+        source = "a = {" + ",".join([str(i) for i in range(200)]) + "}\n"
+        code = compile_with_astcompiler(source, 'exec', space)
+        assert code.co_stacksize < 100
+        w_dict = space.newdict()
+        code.exec_code(space, w_dict, w_dict)
+        assert [space.int_w(w_x)
+                    for w_x in space.unpackiterable(space.getitem(w_dict, space.newtext("a")))] == range(200)
+

From pypy.commits at gmail.com  Wed Mar 27 10:39:53 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 27 Mar 2019 07:39:53 -0700 (PDT)
Subject: [pypy-commit] pypy default: don't use space.str_w
Message-ID: <5c9b8b39.1c69fb81.8608f.3719@mx.google.com>

Author: Carl Friedrich Bolz-Tereick 
Branch: 
Changeset: r96363:9f6768bbafd6
Date: 2019-03-27 15:39 +0100
http://bitbucket.org/pypy/pypy/changeset/9f6768bbafd6/

Log:	don't use space.str_w

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -722,7 +722,7 @@
                 if not e.match(space, space.w_ValueError):
                     raise
                 w_exc_value = e.get_w_value(space)
-                r = "<%s>" % (space.str_w(w_exc_value),)
+                r = "<%s>" % (space.text_w(w_exc_value),)
             else:
                 r = space.text_w(space.repr(w_unicode))
             s = "array('%s', %s)" % (self.typecode, r)

From pypy.commits at gmail.com  Wed Mar 27 10:48:19 2019
From: pypy.commits at gmail.com (arigo)
Date: Wed, 27 Mar 2019 07:48:19 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix comment
Message-ID: <5c9b8d33.1c69fb81.ca2da.445e@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r96364:edb40dc4f969
Date: 2019-03-27 15:46 +0100
http://bitbucket.org/pypy/pypy/changeset/edb40dc4f969/

Log:	fix comment

diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -22,7 +22,7 @@
     prebuilt_gc_objects_are_static_roots = True
     can_usually_pin_objects = False
     object_minimal_size = 0
-    gcflag_extra = 0   # or a real GC flag that is always 0 when not collecting
+    gcflag_extra = 0   # or a dedicated GC flag that the GC initializes to 0
     _totalroots_rpy = 0   # for inspector.py
 
     def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE,

From pypy.commits at gmail.com  Wed Mar 27 10:48:21 2019
From: pypy.commits at gmail.com (arigo)
Date: Wed, 27 Mar 2019 07:48:21 -0700 (PDT)
Subject: [pypy-commit] pypy default: Test and fix for a large list unpacking
Message-ID: <5c9b8d35.1c69fb81.44fa4.18fd@mx.google.com>

Author: Armin Rigo 
Branch: 
Changeset: r96365:f8035bf26e3d
Date: 2019-03-27 15:48 +0100
http://bitbucket.org/pypy/pypy/changeset/f8035bf26e3d/

Log:	Test and fix for a large list unpacking

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -921,7 +921,7 @@
         elt_count = len(l.elts) if l.elts is not None else 0
         if l.ctx == ast.Store:
             self.emit_op_arg(ops.UNPACK_SEQUENCE, elt_count)
-        if elt_count > MAX_STACKDEPTH_CONTAINERS:
+        if elt_count > MAX_STACKDEPTH_CONTAINERS and l.ctx == ast.Load:
             # pushing all the elements would make the stack depth gigantic.
             # build the list incrementally instead
             self.emit_op_arg(ops.BUILD_LIST, 0)
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1229,6 +1229,17 @@
         code.exec_code(space, w_dict, w_dict)
         assert space.unwrap(space.getitem(w_dict, space.newtext("a"))) == range(200)
 
+    def test_list_unpacking(self):
+        space = self.space
+        source = "[" + ",".join(['b%d' % i for i in range(200)]) + "] = a\n"
+        code = compile_with_astcompiler(source, 'exec', space)
+        assert code.co_stacksize == 200   # xxx remains big
+        w_dict = space.newdict()
+        space.setitem(w_dict, space.newtext("a"), space.wrap(range(42, 242)))
+        code.exec_code(space, w_dict, w_dict)
+        assert space.unwrap(space.getitem(w_dict, space.newtext("b0"))) == 42
+        assert space.unwrap(space.getitem(w_dict, space.newtext("b199"))) == 241
+
     def test_set(self):
         space = self.space
         source = "a = {" + ",".join([str(i) for i in range(200)]) + "}\n"

From pypy.commits at gmail.com  Wed Mar 27 16:07:23 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 27 Mar 2019 13:07:23 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge default (including a
 re-implementation of c46e92cbc2f9 for Python3)
Message-ID: <5c9bd7fb.1c69fb81.6d307.6d1e@mx.google.com>

Author: Carl Friedrich Bolz-Tereick 
Branch: py3.6
Changeset: r96366:5521ea7a9241
Date: 2019-03-27 21:00 +0100
http://bitbucket.org/pypy/pypy/changeset/5521ea7a9241/

Log:	merge default (including a re-implementation of c46e92cbc2f9 for
	Python3)

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -21,6 +21,7 @@
     symbols = symtable.SymtableBuilder(space, module, info)
     return TopLevelCodeGenerator(space, module, symbols, info).assemble()
 
+MAX_STACKDEPTH_CONTAINERS = 100
 
 name_ops_default = misc.dict_to_switch({
     ast.Load: ops.LOAD_NAME,
@@ -1222,10 +1223,31 @@
         ifexp.orelse.walkabout(self)
         self.use_next_block(end)
 
-    def _visit_starunpack(self, node, elts, single_op, inner_op, outer_op):
+    def _visit_starunpack(self, node, elts, single_op, inner_op, outer_op, add_op):
         elt_count = len(elts) if elts else 0
         seen_star = 0
         elt_subitems = 0
+        contains_starred = False
+        for i in range(elt_count):
+            elt = elts[i]
+            if isinstance(elt, ast.Starred):
+                contains_starred = True
+                break
+        if elt_count > MAX_STACKDEPTH_CONTAINERS and not contains_starred:
+            tuplecase = False
+            if add_op == -1: # tuples
+                self.emit_op_arg(ops.BUILD_LIST, 0)
+                add_op = ops.LIST_APPEND
+                tuplecase = True
+            else:
+                self.emit_op_arg(single_op, 0)
+            for elt in elts:
+                elt.walkabout(self)
+                self.emit_op_arg(add_op, 1)
+            if tuplecase:
+                self.emit_op_arg(ops.BUILD_TUPLE_UNPACK, 1)
+            return
+
         for i in range(elt_count):
             elt = elts[i]
             is_starred = isinstance(elt, ast.Starred)
@@ -1279,7 +1301,7 @@
         if tup.ctx == ast.Store:
             self._visit_assignment(tup, tup.elts, tup.ctx)
         elif tup.ctx == ast.Load:
-            self._visit_starunpack(tup, tup.elts, ops.BUILD_TUPLE, ops.BUILD_TUPLE, ops.BUILD_TUPLE_UNPACK)
+            self._visit_starunpack(tup, tup.elts, ops.BUILD_TUPLE, ops.BUILD_TUPLE, ops.BUILD_TUPLE_UNPACK, -1)
         else:
             self.visit_sequence(tup.elts)
 
@@ -1288,7 +1310,8 @@
         if l.ctx == ast.Store:
             self._visit_assignment(l, l.elts, l.ctx)
         elif l.ctx == ast.Load:
-            self._visit_starunpack(l, l.elts, ops.BUILD_LIST, ops.BUILD_TUPLE, ops.BUILD_LIST_UNPACK)
+            self._visit_starunpack(
+                l, l.elts, ops.BUILD_LIST, ops.BUILD_TUPLE, ops.BUILD_LIST_UNPACK, ops.LIST_APPEND)
         else:
             self.visit_sequence(l.elts)
 
@@ -1343,7 +1366,7 @@
             is_unpacking = False
 
     def visit_Set(self, s):
-        self._visit_starunpack(s, s.elts, ops.BUILD_SET, ops.BUILD_SET, ops.BUILD_SET_UNPACK)
+        self._visit_starunpack(s, s.elts, ops.BUILD_SET, ops.BUILD_SET, ops.BUILD_SET_UNPACK, ops.SET_ADD)
 
     def visit_Name(self, name):
         self.update_position(name.lineno)
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1543,3 +1543,30 @@
         assert ops.BUILD_TUPLE not in counts
 
 
+
+class TestHugeStackDepths:
+    def run_and_check_stacksize(self, source):
+        space = self.space
+        code = compile_with_astcompiler("a = " + source, 'exec', space)
+        assert code.co_stacksize < 100
+        w_dict = space.newdict()
+        code.exec_code(space, w_dict, w_dict)
+        return space.getitem(w_dict, space.newtext("a"))
+
+    def test_tuple(self):
+        source = "(" + ",".join([str(i) for i in range(200)]) + ")\n"
+        w_res = self.run_and_check_stacksize(source)
+        assert self.space.unwrap(w_res) == tuple(range(200))
+
+    def test_list(self):
+        source = "a = [" + ",".join([str(i) for i in range(200)]) + "]\n"
+        w_res = self.run_and_check_stacksize(source)
+        assert self.space.unwrap(w_res) == range(200)
+
+    def test_set(self):
+        source = "a = {" + ",".join([str(i) for i in range(200)]) + "}\n"
+        w_res = self.run_and_check_stacksize(source)
+        space = self.space
+        assert [space.int_w(w_x)
+                    for w_x in space.unpackiterable(w_res)] == range(200)
+
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -662,14 +662,17 @@
 def wcharpsize2utf8(space, wcharp, size):
     """Safe version of rffi.wcharpsize2utf8.
 
-    Raises app-level rutf8.OutOfRange if any wchar value is outside the valid
+    Raises app-level ValueError if any wchar value is outside the valid
     codepoint range.
     """
     try:
         return rffi.wcharpsize2utf8(wcharp, size)
     except rutf8.OutOfRange as e:
-        raise oefmt(space.w_ValueError,
-            "character %s is not in range [U+0000; U+10ffff]", 'U+%x' % e.code)
+        raise wrap_unicode_out_of_range_error(space, e)
+
+def wrap_unicode_out_of_range_error(space, e):
+    raise oefmt(space.w_ValueError,
+        "character %s is not in range [U+0000; U+10ffff]", 'U+%x' % e.code)
 
 
 # ____________________________________________________________
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -5,6 +5,8 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import interp_attrproperty
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
+from pypy.interpreter.unicodehelper import wcharpsize2utf8
+from pypy.interpreter.unicodehelper import wrap_unicode_out_of_range_error
 
 from rpython.rlib.clibffi import *
 from rpython.rlib.objectmodel import we_are_translated
@@ -598,10 +600,13 @@
     if address == 0:
         return space.w_None
     wcharp_addr = rffi.cast(rffi.CWCHARP, address)
-    if maxlength == -1:
-        s, lgt = rffi.wcharp2utf8(wcharp_addr)
-    else:
-        s, lgt = rffi.wcharp2utf8n(wcharp_addr, maxlength)
+    try:
+        if maxlength == -1:
+            s, lgt = rffi.wcharp2utf8(wcharp_addr)
+        else:
+            s, lgt = rffi.wcharp2utf8n(wcharp_addr, maxlength)
+    except rutf8.OutOfRange as e:
+        raise wrap_unicode_out_of_range_error(space, e)
     return space.newutf8(s, lgt)
 
 @unwrap_spec(address=r_uint, maxlength=int)
@@ -617,7 +622,7 @@
         return wcharp2unicode(space, address)
     elif maxlength < 0:
         maxlength = 0
-    s = rffi.wcharpsize2utf8(rffi.cast(rffi.CWCHARP, address), maxlength)
+    s = wcharpsize2utf8(space, rffi.cast(rffi.CWCHARP, address), maxlength)
     return space.newutf8(s, maxlength)
 
 @unwrap_spec(address=r_uint, newcontent='bufferstr', offset=int, size=int)
diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py
--- a/pypy/module/_rawffi/test/test__rawffi.py
+++ b/pypy/module/_rawffi/test/test__rawffi.py
@@ -1225,6 +1225,23 @@
         lib = _rawffi.CDLL(self.lib_name)
         assert lib.name == self.lib_name
 
+    def test_wcharp2rawunicode(self):
+        import _rawffi
+        A = _rawffi.Array('i')
+        arg = A(1)
+        arg[0] = 0x1234
+        u = _rawffi.wcharp2rawunicode(arg.itemaddress(0))
+        assert u == u'\u1234'
+        u = _rawffi.wcharp2rawunicode(arg.itemaddress(0), 1)
+        assert u == u'\u1234'
+        arg[0] = -1
+        raises(ValueError, _rawffi.wcharp2rawunicode, arg.itemaddress(0))
+        raises(ValueError, _rawffi.wcharp2rawunicode, arg.itemaddress(0), 1)
+        arg[0] = 0x110000
+        raises(ValueError, _rawffi.wcharp2rawunicode, arg.itemaddress(0))
+        raises(ValueError, _rawffi.wcharp2rawunicode, arg.itemaddress(0), 1)
+        arg.free()
+
 
 class AppTestAutoFree:
     spaceconfig = dict(usemodules=['_rawffi', 'struct'])
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -14,6 +14,7 @@
     interp2app, interpindirect2app, unwrap_spec)
 from pypy.interpreter.typedef import (
     GetSetProperty, TypeDef, make_weakref_descr)
+from pypy.interpreter.unicodehelper import wcharpsize2utf8
 
 
 @unwrap_spec(typecode='text')
@@ -509,7 +510,8 @@
             if s < 0:
                 s = 0
             buf = rffi.cast(UNICODE_ARRAY, self._buffer_as_unsigned())
-            return space.newutf8(rffi.wcharpsize2utf8(buf, s), s)
+            utf8 = wcharpsize2utf8(space, buf, s)
+            return space.newutf8(utf8, s)
         else:
             raise oefmt(space.w_ValueError,
                         "tounicode() may only be called on type 'u' arrays")
@@ -767,8 +769,16 @@
         if self.len == 0:
             return space.newtext("array('%s')" % self.typecode)
         elif self.typecode == "u":
-            r = space.repr(self.descr_tounicode(space))
-            s = "array('%s', %s)" % (self.typecode, space.text_w(r))
+            try:
+                w_unicode = self.descr_tounicode(space)
+            except OperationError as e:
+                if not e.match(space, space.w_ValueError):
+                    raise
+                w_exc_value = e.get_w_value(space)
+                r = "<%s>" % (space.text_w(w_exc_value),)
+            else:
+                r = space.text_w(space.repr(w_unicode))
+            s = "array('%s', %s)" % (self.typecode, r)
             return space.newtext(s)
         else:
             r = space.repr(self.descr_tolist(space))
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -893,10 +893,16 @@
         assert repr(mya('i', (1, 2, 3))) == "array('i', [1, 2, 3])"
 
     def test_unicode_outofrange(self):
-        a = self.array('u', u'\x01\u263a\x00\ufeff')
-        b = self.array('u', u'\x01\u263a\x00\ufeff')
+        input_unicode = u'\x01\u263a\x00\ufeff'
+        a = self.array('u', input_unicode)
+        b = self.array('u', input_unicode)
         b.byteswap()
         raises(ValueError, "a != b")
+        assert str(a) == "array('u', %r)" % (input_unicode,)
+        assert str(b) == ("array('u', )")
+        assert a.tounicode() == input_unicode
+        raises(ValueError, b.tounicode)   # doesn't work
 
     def test_weakref(self):
         import weakref
diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py
--- a/pypy/module/gc/referents.py
+++ b/pypy/module/gc/referents.py
@@ -20,7 +20,7 @@
     # inherits from W_Root for internal reasons.  Such instances don't
     # have a typedef at all (or have a null typedef after translation).
     if not we_are_translated():
-        if not hasattr(w_obj, 'typedef'):
+        if getattr(w_obj, 'typedef', None) is None:
             return None
     else:
         if w_obj is None or not w_obj.typedef:
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -1023,7 +1023,7 @@
 def wcharpsize2utf8(w, size):
     """ Helper to convert WCHARP pointer to utf8 in one go.
     Equivalent to wcharpsize2unicode().encode("utf8")
-    Raises ValueError if characters are outside range(0x110000)!
+    Raises rutf8.OutOfRange if characters are outside range(0x110000)!
     """
     from rpython.rlib import rutf8
 
@@ -1033,6 +1033,9 @@
     return s.build()
 
 def wcharp2utf8(w):
+    """
+    Raises rutf8.OutOfRange if characters are outside range(0x110000)!
+    """
     from rpython.rlib import rutf8
 
     s = rutf8.Utf8StringBuilder()
@@ -1043,6 +1046,9 @@
     return s.build(), i
 
 def wcharp2utf8n(w, maxlen):
+    """
+    Raises rutf8.OutOfRange if characters are outside range(0x110000)!
+    """
     from rpython.rlib import rutf8
 
     s = rutf8.Utf8StringBuilder(maxlen)

From pypy.commits at gmail.com  Wed Mar 27 16:07:25 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 27 Mar 2019 13:07:25 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: reorder
Message-ID: <5c9bd7fd.1c69fb81.4418f.9e2e@mx.google.com>

Author: Carl Friedrich Bolz-Tereick 
Branch: py3.6
Changeset: r96367:004ac7c8a23d
Date: 2019-03-27 21:06 +0100
http://bitbucket.org/pypy/pypy/changeset/004ac7c8a23d/

Log:	reorder

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1225,8 +1225,6 @@
 
     def _visit_starunpack(self, node, elts, single_op, inner_op, outer_op, add_op):
         elt_count = len(elts) if elts else 0
-        seen_star = 0
-        elt_subitems = 0
         contains_starred = False
         for i in range(elt_count):
             elt = elts[i]
@@ -1248,6 +1246,8 @@
                 self.emit_op_arg(ops.BUILD_TUPLE_UNPACK, 1)
             return
 
+        seen_star = 0
+        elt_subitems = 0
         for i in range(elt_count):
             elt = elts[i]
             is_starred = isinstance(elt, ast.Starred)

From pypy.commits at gmail.com  Wed Mar 27 17:35:49 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 27 Mar 2019 14:35:49 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: do the same optimization that CPython
 does for (1, 2, 3, *a)
Message-ID: <5c9becb5.1c69fb81.d1c5e.6576@mx.google.com>

Author: Carl Friedrich Bolz-Tereick 
Branch: py3.6
Changeset: r96368:e41b9e56b6cc
Date: 2019-03-27 22:35 +0100
http://bitbucket.org/pypy/pypy/changeset/e41b9e56b6cc/

Log:	do the same optimization that CPython does for (1, 2, 3, *a) (but on
	the AST level)

diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -302,6 +302,9 @@
                 node = tup.elts[i]
                 w_const = node.as_constant()
                 if w_const is None:
+                    new_elts = self._optimize_constant_star_unpacks(tup.elts)
+                    if new_elts is not None:
+                        return ast.Tuple(new_elts, ast.Load, tup.lineno, tup.col_offset)
                     return tup
                 consts_w[i] = w_const
             # intern the string constants packed into the tuple here,
@@ -314,6 +317,62 @@
         w_consts = self.space.newtuple(consts_w)
         return ast.Constant(w_consts, tup.lineno, tup.col_offset)
 
+    def _make_starred_tuple_const(self, consts_w, firstelt):
+        w_consts = self.space.newtuple(consts_w)
+        return ast.Starred(ast.Constant(
+                    w_consts, firstelt.lineno, firstelt.col_offset),
+                ast.Load, firstelt.lineno, firstelt.col_offset)
+
+    def _optimize_constant_star_unpacks(self, elts):
+        # turn (1, 2, 3, *a) into (*(1, 2, 3), *a) with a constant (1, 2, 3)
+        # or similarly, for lists
+        contains_starred = False
+        for i in range(len(elts)):
+            elt = elts[i]
+            if isinstance(elt, ast.Starred):
+                contains_starred = True
+                break
+        if not contains_starred:
+            return None
+        new_elts = []
+        changed = False
+        const_since_last_star_w = []
+        after_last_star_index = 0
+        for i in range(len(elts)):
+            elt = elts[i]
+            if isinstance(elt, ast.Starred):
+                if const_since_last_star_w is not None:
+                    firstelt = elts[after_last_star_index]
+                    new_elts.append(self._make_starred_tuple_const(
+                        const_since_last_star_w, firstelt))
+                    changed = True
+                const_since_last_star_w = []
+                after_last_star_index = i + 1
+                new_elts.append(elt)
+            elif const_since_last_star_w is not None:
+                w_const = elt.as_constant()
+                if w_const is None:
+                    new_elts.extend(elts[after_last_star_index:i + 1])
+                    const_since_last_star_w = None
+                else:
+                    const_since_last_star_w.append(w_const)
+            else:
+                new_elts.append(elt)
+        if after_last_star_index != len(elts) and const_since_last_star_w is not None:
+            firstelt = elts[after_last_star_index]
+            new_elts.append(self._make_starred_tuple_const(
+                const_since_last_star_w, firstelt))
+            changed = True
+        if changed:
+            return new_elts
+
+    def visit_List(self, l):
+        if l.ctx == ast.Load and l.elts:
+            new_elts = self._optimize_constant_star_unpacks(l.elts)
+            if new_elts:
+                return ast.List(new_elts, ast.Load, l.lineno, l.col_offset)
+        return l
+
     def visit_Subscript(self, subs):
         if subs.ctx == ast.Load:
             w_obj = subs.value.as_constant()
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1099,14 +1099,23 @@
                 return a, b, c
         """
         yield self.st, func, "f()", (1, [2, 3], 4)
+
+    def test_unpacking_while_building(self):
         func = """def f():
             b = [4,5,6]
-            c = 7
-            a = [*b, c]
+            a = (*b, 7)
+            return a
+        """
+        yield self.st, func, "f()", (4, 5, 6, 7)
+
+        func = """def f():
+            b = [4,5,6]
+            a = [*b, 7]
             return a
         """
         yield self.st, func, "f()", [4, 5, 6, 7]
 
+
     def test_extended_unpacking_fail(self):
         exc = py.test.raises(SyntaxError, self.simple_test, "*a, *b = [1, 2]",
                              None, None).value
@@ -1542,6 +1551,31 @@
         counts = self.count_instructions(source)
         assert ops.BUILD_TUPLE not in counts
 
+    def test_constant_tuples_star(self):
+        source = """def f(a, c):
+            return (u"a", 1, *a, 3, 5, 3, *c)
+        """
+        counts = self.count_instructions(source)
+        assert ops.BUILD_TUPLE not in counts
+
+        source = """def f(a, c, d):
+            return (u"a", 1, *a, c, 1, *d, 1, 2, 3)
+        """
+        counts = self.count_instructions(source)
+        assert counts[ops.BUILD_TUPLE] == 1
+
+    def test_constant_list_star(self):
+        source = """def f(a, c):
+            return [u"a", 1, *a, 3, 5, 3, *c]
+        """
+        counts = self.count_instructions(source)
+        assert ops.BUILD_TUPLE not in counts
+
+        source = """def f(a, c, d):
+            return [u"a", 1, *a, c, 1, *d, 1, 2, 3]
+        """
+        counts = self.count_instructions(source)
+        assert counts[ops.BUILD_TUPLE] == 1
 
 
 class TestHugeStackDepths:

From pypy.commits at gmail.com  Wed Mar 27 18:01:23 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 27 Mar 2019 15:01:23 -0700 (PDT)
Subject: [pypy-commit] pypy default: (cfbolz,
 arigo giving the idea) to construct gigantic tuples in constant
 stack
Message-ID: <5c9bf2b3.1c69fb81.8608f.cd84@mx.google.com>

Author: Carl Friedrich Bolz-Tereick 
Branch: 
Changeset: r96369:f6a9c3de9c7d
Date: 2019-03-27 22:51 +0100
http://bitbucket.org/pypy/pypy/changeset/f6a9c3de9c7d/

Log:	(cfbolz, arigo giving the idea) to construct gigantic tuples in
	constant stack space, first build a list, then use
	().__class__(list) to turn it into a tuple. bit of a hack, but
	calling 'tuple' doesn't work if somebody overrides the global name
	'tuple'

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -912,6 +912,20 @@
         elt_count = len(tup.elts) if tup.elts is not None else 0
         if tup.ctx == ast.Store:
             self.emit_op_arg(ops.UNPACK_SEQUENCE, elt_count)
+        if tup.ctx == ast.Load and elt_count > MAX_STACKDEPTH_CONTAINERS:
+            # we need a complete hack to build a new tuple from the list
+            # ().__class__(l)
+            empty_index = self.add_const(self.space.newtuple([]))
+            self.emit_op_arg(ops.LOAD_CONST, empty_index)
+            self.emit_op_name(ops.LOAD_ATTR, self.names, '__class__')
+
+            self.emit_op_arg(ops.BUILD_LIST, 0)
+            for element in tup.elts:
+                element.walkabout(self)
+                self.emit_op_arg(ops.LIST_APPEND, 1)
+
+            self.emit_op_arg(ops.CALL_FUNCTION, 1)
+            return
         self.visit_sequence(tup.elts)
         if tup.ctx == ast.Load:
             self.emit_op_arg(ops.BUILD_TUPLE, elt_count)
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1220,22 +1220,27 @@
 
 
 class TestHugeStackDepths:
-    def test_list(self):
+    def run_and_check_stacksize(self, source):
         space = self.space
-        source = "a = [" + ",".join([str(i) for i in range(200)]) + "]\n"
-        code = compile_with_astcompiler(source, 'exec', space)
+        code = compile_with_astcompiler("a = " + source, 'exec', space)
         assert code.co_stacksize < 100
         w_dict = space.newdict()
         code.exec_code(space, w_dict, w_dict)
-        assert space.unwrap(space.getitem(w_dict, space.newtext("a"))) == range(200)
+        return space.getitem(w_dict, space.newtext("a"))
+
+    def test_tuple(self):
+        source = "(" + ",".join([str(i) for i in range(200)]) + ")\n"
+        w_res = self.run_and_check_stacksize(source)
+        assert self.space.unwrap(w_res) == tuple(range(200))
+
+    def test_list(self):
+        source = "a = [" + ",".join([str(i) for i in range(200)]) + "]\n"
+        w_res = self.run_and_check_stacksize(source)
+        assert self.space.unwrap(w_res) == range(200)
 
     def test_set(self):
+        source = "a = {" + ",".join([str(i) for i in range(200)]) + "}\n"
+        w_res = self.run_and_check_stacksize(source)
         space = self.space
-        source = "a = {" + ",".join([str(i) for i in range(200)]) + "}\n"
-        code = compile_with_astcompiler(source, 'exec', space)
-        assert code.co_stacksize < 100
-        w_dict = space.newdict()
-        code.exec_code(space, w_dict, w_dict)
         assert [space.int_w(w_x)
-                    for w_x in space.unpackiterable(space.getitem(w_dict, space.newtext("a")))] == range(200)
-
+                    for w_x in space.unpackiterable(w_res)] == range(200)

From pypy.commits at gmail.com  Wed Mar 27 18:01:25 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 27 Mar 2019 15:01:25 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge default (no changes,
 the 3.6 version already worked)
Message-ID: <5c9bf2b5.1c69fb81.57e81.fb37@mx.google.com>

Author: Carl Friedrich Bolz-Tereick 
Branch: py3.6
Changeset: r96370:1790f4267a4b
Date: 2019-03-27 22:56 +0100
http://bitbucket.org/pypy/pypy/changeset/1790f4267a4b/

Log:	merge default (no changes, the 3.6 version already worked)

diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1603,4 +1603,3 @@
         space = self.space
         assert [space.int_w(w_x)
                     for w_x in space.unpackiterable(w_res)] == range(200)
-

From pypy.commits at gmail.com  Wed Mar 27 18:01:27 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 27 Mar 2019 15:01:27 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <5c9bf2b7.1c69fb81.f217f.2f6a@mx.google.com>

Author: Carl Friedrich Bolz-Tereick 
Branch: 
Changeset: r96371:24e79a787bd8
Date: 2019-03-27 23:00 +0100
http://bitbucket.org/pypy/pypy/changeset/24e79a787bd8/

Log:	merge heads

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -912,6 +912,20 @@
         elt_count = len(tup.elts) if tup.elts is not None else 0
         if tup.ctx == ast.Store:
             self.emit_op_arg(ops.UNPACK_SEQUENCE, elt_count)
+        if tup.ctx == ast.Load and elt_count > MAX_STACKDEPTH_CONTAINERS:
+            # we need a complete hack to build a new tuple from the list
+            # ().__class__(l)
+            empty_index = self.add_const(self.space.newtuple([]))
+            self.emit_op_arg(ops.LOAD_CONST, empty_index)
+            self.emit_op_name(ops.LOAD_ATTR, self.names, '__class__')
+
+            self.emit_op_arg(ops.BUILD_LIST, 0)
+            for element in tup.elts:
+                element.walkabout(self)
+                self.emit_op_arg(ops.LIST_APPEND, 1)
+
+            self.emit_op_arg(ops.CALL_FUNCTION, 1)
+            return
         self.visit_sequence(tup.elts)
         if tup.ctx == ast.Load:
             self.emit_op_arg(ops.BUILD_TUPLE, elt_count)
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1220,14 +1220,23 @@
 
 
 class TestHugeStackDepths:
-    def test_list(self):
+    def run_and_check_stacksize(self, source):
         space = self.space
-        source = "a = [" + ",".join([str(i) for i in range(200)]) + "]\n"
-        code = compile_with_astcompiler(source, 'exec', space)
+        code = compile_with_astcompiler("a = " + source, 'exec', space)
         assert code.co_stacksize < 100
         w_dict = space.newdict()
         code.exec_code(space, w_dict, w_dict)
-        assert space.unwrap(space.getitem(w_dict, space.newtext("a"))) == range(200)
+        return space.getitem(w_dict, space.newtext("a"))
+
+    def test_tuple(self):
+        source = "(" + ",".join([str(i) for i in range(200)]) + ")\n"
+        w_res = self.run_and_check_stacksize(source)
+        assert self.space.unwrap(w_res) == tuple(range(200))
+
+    def test_list(self):
+        source = "a = [" + ",".join([str(i) for i in range(200)]) + "]\n"
+        w_res = self.run_and_check_stacksize(source)
+        assert self.space.unwrap(w_res) == range(200)
 
     def test_list_unpacking(self):
         space = self.space
@@ -1241,12 +1250,8 @@
         assert space.unwrap(space.getitem(w_dict, space.newtext("b199"))) == 241
 
     def test_set(self):
+        source = "a = {" + ",".join([str(i) for i in range(200)]) + "}\n"
+        w_res = self.run_and_check_stacksize(source)
         space = self.space
-        source = "a = {" + ",".join([str(i) for i in range(200)]) + "}\n"
-        code = compile_with_astcompiler(source, 'exec', space)
-        assert code.co_stacksize < 100
-        w_dict = space.newdict()
-        code.exec_code(space, w_dict, w_dict)
         assert [space.int_w(w_x)
-                    for w_x in space.unpackiterable(space.getitem(w_dict, space.newtext("a")))] == range(200)
-
+                    for w_x in space.unpackiterable(w_res)] == range(200)

From pypy.commits at gmail.com  Thu Mar 28 03:54:18 2019
From: pypy.commits at gmail.com (andrewjlawrence)
Date: Thu, 28 Mar 2019 00:54:18 -0700 (PDT)
Subject: [pypy-commit] pypy winoverlapped: Fixed exception types produced by
 overlapped.getresult, reverted rwin32.py
Message-ID: <5c9c7daa.1c69fb81.dcdb7.c0af@mx.google.com>

Author: andrewjlawrence
Branch: winoverlapped
Changeset: r96372:79630c7e2718
Date: 2019-03-28 07:46 +0000
http://bitbucket.org/pypy/pypy/changeset/79630c7e2718/

Log:	Fixed exception types produced by overlapped.getresult, reverted
	rwin32.py

diff --git a/lib_pypy/_overlapped.py b/lib_pypy/_overlapped.py
--- a/lib_pypy/_overlapped.py
+++ b/lib_pypy/_overlapped.py
@@ -207,7 +207,7 @@
 
         if err != _winapi.ERROR_SUCCESS and err != _winapi.ERROR_MORE_DATA:
             if not (err == _winapi.ERROR_BROKEN_PIPE and (self.type in [OverlappedType.TYPE_READ, OverlappedType.TYPE_READINTO])):
-                raise _winapi._WinError()
+                SetFromWindowsErr(err)
 
         if self.type == OverlappedType.TYPE_READ:
             return _ffi.unpack(self.read_buffer, transferred[0])
@@ -578,8 +578,16 @@
 
 # In CPython this function converts a windows error into a python object
 # Not sure what we should do here.
-def SetFromWindowsErr(error):
-    return error
+def SetFromWindowsErr(err):
+    if err == _winapi.ERROR_CONNECTION_REFUSED:
+        type = ConnectionRefusedError;
+    elif err == _winapi.ERROR_CONNECTION_ABORTED:
+        type = ConnectionAbortedError;
+    else:
+        type = WindowsError;
+
+    return _winapi._WinError(type);
+
 
 def HasOverlappedIoCompleted(overlapped):
     return (overlapped.Internal != STATUS_PENDING)
diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py
--- a/lib_pypy/_winapi.py
+++ b/lib_pypy/_winapi.py
@@ -19,9 +19,9 @@
 # Now the _subprocess module implementation
 
 
-def _WinError():
+def _WinError(type=WindowsError):
     code, message = _ffi.getwinerror()
-    excep = WindowsError(None, message, None ,code)
+    excep = type(None, message, None ,code)
     raise excep
 
 def _int2handle(val):
@@ -303,7 +303,7 @@
 ERROR_OPERATION_ABORTED = 995
 ERROR_IO_INCOMPLETE     = 996
 ERROR_IO_PENDING        = 997
-
+ERROR_CONNECTION_REFUSED = 1225
 
 PIPE_ACCESS_INBOUND = 0x00000001
 PIPE_ACCESS_OUTBOUND = 0x00000002
diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -261,7 +261,7 @@
                 132: 13, 145: 41, 158: 13, 161: 2, 164: 11, 167: 13, 183: 17,
                 188: 8, 189: 8, 190: 8, 191: 8, 192: 8, 193: 8, 194: 8,
                 195: 8, 196: 8, 197: 8, 198: 8, 199: 8, 200: 8, 201: 8,
-                202: 8, 206: 2, 215: 11, 232: 32, 267: 20, 1816: 12, 1225: 111,
+                202: 8, 206: 2, 215: 11, 232: 32, 267: 20, 1816: 12, 
                 }
         return errors, errno.EINVAL
 

From pypy.commits at gmail.com  Thu Mar 28 03:54:20 2019
From: pypy.commits at gmail.com (andrewjlawrence)
Date: Thu, 28 Mar 2019 00:54:20 -0700 (PDT)
Subject: [pypy-commit] pypy winoverlapped: Add connection aborted error code.
Message-ID: <5c9c7dac.1c69fb81.c0ab7.2bfd@mx.google.com>

Author: andrewjlawrence
Branch: winoverlapped
Changeset: r96373:60b0c39477d5
Date: 2019-03-28 07:52 +0000
http://bitbucket.org/pypy/pypy/changeset/60b0c39477d5/

Log:	Add connection aborted error code.

diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py
--- a/lib_pypy/_winapi.py
+++ b/lib_pypy/_winapi.py
@@ -304,6 +304,7 @@
 ERROR_IO_INCOMPLETE     = 996
 ERROR_IO_PENDING        = 997
 ERROR_CONNECTION_REFUSED = 1225
+ERROR_CONNECTION_ABORTED = 1236
 
 PIPE_ACCESS_INBOUND = 0x00000001
 PIPE_ACCESS_OUTBOUND = 0x00000002

From pypy.commits at gmail.com  Thu Mar 28 06:22:59 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 28 Mar 2019 03:22:59 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: construct huge dicts in constant stack
 space as well
Message-ID: <5c9ca083.1c69fb81.ca2da.3e83@mx.google.com>

Author: Carl Friedrich Bolz-Tereick 
Branch: py3.6
Changeset: r96374:5ea8098b2f2c
Date: 2019-03-28 11:22 +0100
http://bitbucket.org/pypy/pypy/changeset/5ea8098b2f2c/

Log:	construct huge dicts in constant stack space as well

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1322,6 +1322,21 @@
         is_unpacking = False
         all_constant_keys_w = None
         if d.values:
+            unpacking_anywhere = False
+            for key in d.keys:
+                if key is None:
+                    unpacking_anywhere = True
+                    break
+            if not unpacking_anywhere and len(d.keys) > MAX_STACKDEPTH_CONTAINERS:
+                # do it in a small amount of stack
+                self.emit_op_arg(ops.BUILD_MAP, 0)
+                for i in range(len(d.values)):
+                    d.values[i].walkabout(self)
+                    key = d.keys[i]
+                    assert key is not None
+                    key.walkabout(self)
+                    self.emit_op_arg(ops.MAP_ADD, 1)
+                return
             if len(d.keys) < 0xffff:
                 all_constant_keys_w = []
                 for key in d.keys:
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1593,13 +1593,18 @@
         assert self.space.unwrap(w_res) == tuple(range(200))
 
     def test_list(self):
-        source = "a = [" + ",".join([str(i) for i in range(200)]) + "]\n"
+        source = "[" + ",".join([str(i) for i in range(200)]) + "]\n"
         w_res = self.run_and_check_stacksize(source)
         assert self.space.unwrap(w_res) == range(200)
 
     def test_set(self):
-        source = "a = {" + ",".join([str(i) for i in range(200)]) + "}\n"
+        source = "{" + ",".join([str(i) for i in range(200)]) + "}\n"
         w_res = self.run_and_check_stacksize(source)
         space = self.space
         assert [space.int_w(w_x)
                     for w_x in space.unpackiterable(w_res)] == range(200)
+
+    def test_dict(self):
+        source = "{" + ",".join(['%s: None' % (i, ) for i in range(200)]) + "}\n"
+        w_res = self.run_and_check_stacksize(source)
+        assert self.space.unwrap(w_res) == dict.fromkeys(range(200))

From pypy.commits at gmail.com  Thu Mar 28 06:31:46 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 28 Mar 2019 03:31:46 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: fix translation
Message-ID: <5c9ca292.1c69fb81.ccad5.b9a5@mx.google.com>

Author: Carl Friedrich Bolz-Tereick 
Branch: py3.6
Changeset: r96375:43bb1c622881
Date: 2019-03-28 11:30 +0100
http://bitbucket.org/pypy/pypy/changeset/43bb1c622881/

Log:	fix translation

diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -318,7 +318,7 @@
         return ast.Constant(w_consts, tup.lineno, tup.col_offset)
 
     def _make_starred_tuple_const(self, consts_w, firstelt):
-        w_consts = self.space.newtuple(consts_w)
+        w_consts = self.space.newtuple(consts_w[:])
         return ast.Starred(ast.Constant(
                     w_consts, firstelt.lineno, firstelt.col_offset),
                 ast.Load, firstelt.lineno, firstelt.col_offset)

From pypy.commits at gmail.com  Thu Mar 28 06:42:31 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 28 Mar 2019 03:42:31 -0700 (PDT)
Subject: [pypy-commit] pypy default: the test that I think we should figure
 out how to pass
Message-ID: <5c9ca517.1c69fb81.fdfde.42ff@mx.google.com>

Author: Carl Friedrich Bolz-Tereick 
Branch: 
Changeset: r96376:3929440c517f
Date: 2019-03-28 11:38 +0100
http://bitbucket.org/pypy/pypy/changeset/3929440c517f/

Log:	the test that I think we should figure out how to pass

diff --git a/rpython/jit/metainterp/test/test_virtualizable.py b/rpython/jit/metainterp/test/test_virtualizable.py
--- a/rpython/jit/metainterp/test/test_virtualizable.py
+++ b/rpython/jit/metainterp/test/test_virtualizable.py
@@ -542,6 +542,41 @@
         self.check_resops(getfield_gc_r=1, getarrayitem_gc_i=4,
                           getfield_gc_i=1)
 
+    @py.test.mark.xfail
+    def test_virtualizable_with_array_huge(self):
+        myjitdriver = JitDriver(greens = [], reds = ['n', 'x', 'frame'],
+                                virtualizables = ['frame'])
+
+        class Frame(object):
+            _virtualizable_ = ['l[*]', 's']
+
+            def __init__(self, l, s):
+                self.l = l
+                self.s = s
+
+        def f(n, a):
+            frame = Frame([a] * 1024, 0) # make a huge frame
+            x = 0
+            while n > 0:
+                myjitdriver.can_enter_jit(frame=frame, n=n, x=x)
+                myjitdriver.jit_merge_point(frame=frame, n=n, x=x)
+                frame.s = promote(frame.s)
+                n -= 1
+                s = frame.s
+                assert s >= 0
+                x += frame.l[s]
+                frame.s += 1
+                s = frame.s
+                assert s >= 0
+                x += frame.l[s]
+                x += len(frame.l)
+                frame.s -= 1
+            return x
+
+        res = self.meta_interp(f, [50, 1], listops=True)
+        # should stop giving up to compile, eventually
+        assert get_stats().aborted_count < 6
+
     def test_subclass_of_virtualizable(self):
         myjitdriver = JitDriver(greens = [], reds = ['frame'],
                                 virtualizables = ['frame'])

From pypy.commits at gmail.com  Thu Mar 28 08:29:44 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 28 Mar 2019 05:29:44 -0700 (PDT)
Subject: [pypy-commit] pypy default: kill identical helper function
Message-ID: <5c9cbe38.1c69fb81.a0e04.f2f7@mx.google.com>

Author: Carl Friedrich Bolz-Tereick 
Branch: 
Changeset: r96380:a8deefd689d8
Date: 2019-03-28 13:28 +0100
http://bitbucket.org/pypy/pypy/changeset/a8deefd689d8/

Log:	kill identical helper function

diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py
--- a/pypy/objspace/std/celldict.py
+++ b/pypy/objspace/std/celldict.py
@@ -10,17 +10,12 @@
     DictStrategy, ObjectDictStrategy, _never_equal_to_string,
     create_iterator_classes)
 from pypy.objspace.std.typeobject import (
-    MutableCell, IntMutableCell, ObjectMutableCell, write_cell)
+    MutableCell, IntMutableCell, ObjectMutableCell, write_cell, unwrap_cell)
 
 
 class VersionTag(object):
     pass
 
-def unwrap_cell(space, w_value):
-    if isinstance(w_value, MutableCell):
-        return w_value.unwrap_cell(space)
-    return w_value
-
 
 def _wrapkey(space, key):
     return space.newtext(key)

From pypy.commits at gmail.com  Thu Mar 28 13:06:54 2019
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 28 Mar 2019 10:06:54 -0700 (PDT)
Subject: [pypy-commit] pypy default: it passes like this
Message-ID: <5c9cff2e.1c69fb81.3859f.b41c@mx.google.com>

Author: Carl Friedrich Bolz-Tereick 
Branch: 
Changeset: r96382:439f4942e2bd
Date: 2019-03-28 18:06 +0100
http://bitbucket.org/pypy/pypy/changeset/439f4942e2bd/

Log:	it passes like this

diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -109,7 +109,6 @@
         self.check_trace_count(1)
         self.check_simple_loop(int_mul=1)
 
-    @py.test.mark.xfail
     def test_rutf8(self):
         from rpython.rlib import rutf8, jit
         class U(object):
@@ -133,7 +132,8 @@
             x = str(a)
             u = U(x, len(x))
             st = u._get_index_storage()
-            return st[0].baseindex
+            return rutf8.codepoint_index_at_byte_position(
+                u.u, st, 1)
 
         self.interp_operations(m, [123232])
 

From pypy.commits at gmail.com  Thu Mar 28 18:20:24 2019
From: pypy.commits at gmail.com (andrewjlawrence)
Date: Thu, 28 Mar 2019 15:20:24 -0700 (PDT)
Subject: [pypy-commit] pypy winoverlapped: Correct GetLastError module.
Message-ID: <5c9d48a8.1c69fb81.aed9d.121b@mx.google.com>

Author: andrewjlawrence
Branch: winoverlapped
Changeset: r96385:c91d95fe08ff
Date: 2019-03-28 22:16 +0000
http://bitbucket.org/pypy/pypy/changeset/c91d95fe08ff/

Log:	Correct GetLastError module.

diff --git a/lib_pypy/_overlapped.py b/lib_pypy/_overlapped.py
--- a/lib_pypy/_overlapped.py
+++ b/lib_pypy/_overlapped.py
@@ -225,7 +225,7 @@
         if not HasOverlappedIoCompleted(self.overlapped[0]):
             ### If we are to support xp we will need to dynamically load the below method
             result = _kernel32.CancelIoEx(self.handle, self.overlapped)
-        if (not result and _winapi.GetLastError() != _winapi.ERROR_NOT_FOUND):
+        if (not result and _kernel32.GetLastError() != _winapi.ERROR_NOT_FOUND):
             raise _winapi._WinError()
      
     def WSARecv(self ,handle, size, flags):
@@ -356,7 +356,7 @@
         if ret:
             self.error = _winapi.ERROR_SUCCESS
         else:
-            self.error = _winapi.GetLastError()
+            self.error = _kernel32.GetLastError()
 
         if self.error == _winapi.ERROR_SUCCESS or self.error == _winapi.ERROR_IO_PENDING:
             return None

From pypy.commits at gmail.com  Thu Mar 28 18:20:26 2019
From: pypy.commits at gmail.com (andrewjlawrence)
Date: Thu, 28 Mar 2019 15:20:26 -0700 (PDT)
Subject: [pypy-commit] pypy winoverlapped: Merged py3.6 changes
Message-ID: <5c9d48aa.1c69fb81.78623.35c2@mx.google.com>

Author: andrewjlawrence
Branch: winoverlapped
Changeset: r96386:6c5e88b3724e
Date: 2019-03-28 22:18 +0000
http://bitbucket.org/pypy/pypy/changeset/6c5e88b3724e/

Log:	Merged py3.6 changes

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -21,6 +21,7 @@
     symbols = symtable.SymtableBuilder(space, module, info)
     return TopLevelCodeGenerator(space, module, symbols, info).assemble()
 
+MAX_STACKDEPTH_CONTAINERS = 100
 
 name_ops_default = misc.dict_to_switch({
     ast.Load: ops.LOAD_NAME,
@@ -1222,8 +1223,29 @@
         ifexp.orelse.walkabout(self)
         self.use_next_block(end)
 
-    def _visit_starunpack(self, node, elts, single_op, inner_op, outer_op):
+    def _visit_starunpack(self, node, elts, single_op, inner_op, outer_op, add_op):
         elt_count = len(elts) if elts else 0
+        contains_starred = False
+        for i in range(elt_count):
+            elt = elts[i]
+            if isinstance(elt, ast.Starred):
+                contains_starred = True
+                break
+        if elt_count > MAX_STACKDEPTH_CONTAINERS and not contains_starred:
+            tuplecase = False
+            if add_op == -1: # tuples
+                self.emit_op_arg(ops.BUILD_LIST, 0)
+                add_op = ops.LIST_APPEND
+                tuplecase = True
+            else:
+                self.emit_op_arg(single_op, 0)
+            for elt in elts:
+                elt.walkabout(self)
+                self.emit_op_arg(add_op, 1)
+            if tuplecase:
+                self.emit_op_arg(ops.BUILD_TUPLE_UNPACK, 1)
+            return
+
         seen_star = 0
         elt_subitems = 0
         for i in range(elt_count):
@@ -1279,7 +1301,7 @@
         if tup.ctx == ast.Store:
             self._visit_assignment(tup, tup.elts, tup.ctx)
         elif tup.ctx == ast.Load:
-            self._visit_starunpack(tup, tup.elts, ops.BUILD_TUPLE, ops.BUILD_TUPLE, ops.BUILD_TUPLE_UNPACK)
+            self._visit_starunpack(tup, tup.elts, ops.BUILD_TUPLE, ops.BUILD_TUPLE, ops.BUILD_TUPLE_UNPACK, -1)
         else:
             self.visit_sequence(tup.elts)
 
@@ -1288,7 +1310,8 @@
         if l.ctx == ast.Store:
             self._visit_assignment(l, l.elts, l.ctx)
         elif l.ctx == ast.Load:
-            self._visit_starunpack(l, l.elts, ops.BUILD_LIST, ops.BUILD_TUPLE, ops.BUILD_LIST_UNPACK)
+            self._visit_starunpack(
+                l, l.elts, ops.BUILD_LIST, ops.BUILD_TUPLE, ops.BUILD_LIST_UNPACK, ops.LIST_APPEND)
         else:
             self.visit_sequence(l.elts)
 
@@ -1299,6 +1322,21 @@
         is_unpacking = False
         all_constant_keys_w = None
         if d.values:
+            unpacking_anywhere = False
+            for key in d.keys:
+                if key is None:
+                    unpacking_anywhere = True
+                    break
+            if not unpacking_anywhere and len(d.keys) > MAX_STACKDEPTH_CONTAINERS:
+                # do it in a small amount of stack
+                self.emit_op_arg(ops.BUILD_MAP, 0)
+                for i in range(len(d.values)):
+                    d.values[i].walkabout(self)
+                    key = d.keys[i]
+                    assert key is not None
+                    key.walkabout(self)
+                    self.emit_op_arg(ops.MAP_ADD, 1)
+                return
             if len(d.keys) < 0xffff:
                 all_constant_keys_w = []
                 for key in d.keys:
@@ -1343,7 +1381,7 @@
             is_unpacking = False
 
     def visit_Set(self, s):
-        self._visit_starunpack(s, s.elts, ops.BUILD_SET, ops.BUILD_SET, ops.BUILD_SET_UNPACK)
+        self._visit_starunpack(s, s.elts, ops.BUILD_SET, ops.BUILD_SET, ops.BUILD_SET_UNPACK, ops.SET_ADD)
 
     def visit_Name(self, name):
         self.update_position(name.lineno)
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -302,6 +302,9 @@
                 node = tup.elts[i]
                 w_const = node.as_constant()
                 if w_const is None:
+                    new_elts = self._optimize_constant_star_unpacks(tup.elts)
+                    if new_elts is not None:
+                        return ast.Tuple(new_elts, ast.Load, tup.lineno, tup.col_offset)
                     return tup
                 consts_w[i] = w_const
             # intern the string constants packed into the tuple here,
@@ -314,6 +317,62 @@
         w_consts = self.space.newtuple(consts_w)
         return ast.Constant(w_consts, tup.lineno, tup.col_offset)
 
+    def _make_starred_tuple_const(self, consts_w, firstelt):
+        w_consts = self.space.newtuple(consts_w[:])
+        return ast.Starred(ast.Constant(
+                    w_consts, firstelt.lineno, firstelt.col_offset),
+                ast.Load, firstelt.lineno, firstelt.col_offset)
+
+    def _optimize_constant_star_unpacks(self, elts):
+        # turn (1, 2, 3, *a) into (*(1, 2, 3), *a) with a constant (1, 2, 3)
+        # or similarly, for lists
+        contains_starred = False
+        for i in range(len(elts)):
+            elt = elts[i]
+            if isinstance(elt, ast.Starred):
+                contains_starred = True
+                break
+        if not contains_starred:
+            return None
+        new_elts = []
+        changed = False
+        const_since_last_star_w = []
+        after_last_star_index = 0
+        for i in range(len(elts)):
+            elt = elts[i]
+            if isinstance(elt, ast.Starred):
+                if const_since_last_star_w is not None:
+                    firstelt = elts[after_last_star_index]
+                    new_elts.append(self._make_starred_tuple_const(
+                        const_since_last_star_w, firstelt))
+                    changed = True
+                const_since_last_star_w = []
+                after_last_star_index = i + 1
+                new_elts.append(elt)
+            elif const_since_last_star_w is not None:
+                w_const = elt.as_constant()
+                if w_const is None:
+                    new_elts.extend(elts[after_last_star_index:i + 1])
+                    const_since_last_star_w = None
+                else:
+                    const_since_last_star_w.append(w_const)
+            else:
+                new_elts.append(elt)
+        if after_last_star_index != len(elts) and const_since_last_star_w is not None:
+            firstelt = elts[after_last_star_index]
+            new_elts.append(self._make_starred_tuple_const(
+                const_since_last_star_w, firstelt))
+            changed = True
+        if changed:
+            return new_elts
+
+    def visit_List(self, l):
+        if l.ctx == ast.Load and l.elts:
+            new_elts = self._optimize_constant_star_unpacks(l.elts)
+            if new_elts:
+                return ast.List(new_elts, ast.Load, l.lineno, l.col_offset)
+        return l
+
     def visit_Subscript(self, subs):
         if subs.ctx == ast.Load:
             w_obj = subs.value.as_constant()
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1099,14 +1099,23 @@
                 return a, b, c
         """
         yield self.st, func, "f()", (1, [2, 3], 4)
+
+    def test_unpacking_while_building(self):
         func = """def f():
             b = [4,5,6]
-            c = 7
-            a = [*b, c]
+            a = (*b, 7)
+            return a
+        """
+        yield self.st, func, "f()", (4, 5, 6, 7)
+
+        func = """def f():
+            b = [4,5,6]
+            a = [*b, 7]
             return a
         """
         yield self.st, func, "f()", [4, 5, 6, 7]
 
+
     def test_extended_unpacking_fail(self):
         exc = py.test.raises(SyntaxError, self.simple_test, "*a, *b = [1, 2]",
                              None, None).value
@@ -1542,4 +1551,60 @@
         counts = self.count_instructions(source)
         assert ops.BUILD_TUPLE not in counts
 
+    def test_constant_tuples_star(self):
+        source = """def f(a, c):
+            return (u"a", 1, *a, 3, 5, 3, *c)
+        """
+        counts = self.count_instructions(source)
+        assert ops.BUILD_TUPLE not in counts
 
+        source = """def f(a, c, d):
+            return (u"a", 1, *a, c, 1, *d, 1, 2, 3)
+        """
+        counts = self.count_instructions(source)
+        assert counts[ops.BUILD_TUPLE] == 1
+
+    def test_constant_list_star(self):
+        source = """def f(a, c):
+            return [u"a", 1, *a, 3, 5, 3, *c]
+        """
+        counts = self.count_instructions(source)
+        assert ops.BUILD_TUPLE not in counts
+
+        source = """def f(a, c, d):
+            return [u"a", 1, *a, c, 1, *d, 1, 2, 3]
+        """
+        counts = self.count_instructions(source)
+        assert counts[ops.BUILD_TUPLE] == 1
+
+
+class TestHugeStackDepths:
+    def run_and_check_stacksize(self, source):
+        space = self.space
+        code = compile_with_astcompiler("a = " + source, 'exec', space)
+        assert code.co_stacksize < 100
+        w_dict = space.newdict()
+        code.exec_code(space, w_dict, w_dict)
+        return space.getitem(w_dict, space.newtext("a"))
+
+    def test_tuple(self):
+        source = "(" + ",".join([str(i) for i in range(200)]) + ")\n"
+        w_res = self.run_and_check_stacksize(source)
+        assert self.space.unwrap(w_res) == tuple(range(200))
+
+    def test_list(self):
+        source = "[" + ",".join([str(i) for i in range(200)]) + "]\n"
+        w_res = self.run_and_check_stacksize(source)
+        assert self.space.unwrap(w_res) == range(200)
+
+    def test_set(self):
+        source = "{" + ",".join([str(i) for i in range(200)]) + "}\n"
+        w_res = self.run_and_check_stacksize(source)
+        space = self.space
+        assert [space.int_w(w_x)
+                    for w_x in space.unpackiterable(w_res)] == range(200)
+
+    def test_dict(self):
+        source = "{" + ",".join(['%s: None' % (i, ) for i in range(200)]) + "}\n"
+        w_res = self.run_and_check_stacksize(source)
+        assert self.space.unwrap(w_res) == dict.fromkeys(range(200))
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -662,14 +662,17 @@
 def wcharpsize2utf8(space, wcharp, size):
     """Safe version of rffi.wcharpsize2utf8.
 
-    Raises app-level rutf8.OutOfRange if any wchar value is outside the valid
+    Raises app-level ValueError if any wchar value is outside the valid
     codepoint range.
     """
     try:
         return rffi.wcharpsize2utf8(wcharp, size)
     except rutf8.OutOfRange as e:
-        raise oefmt(space.w_ValueError,
-            "character %s is not in range [U+0000; U+10ffff]", 'U+%x' % e.code)
+        raise wrap_unicode_out_of_range_error(space, e)
+
+def wrap_unicode_out_of_range_error(space, e):
+    raise oefmt(space.w_ValueError,
+        "character %s is not in range [U+0000; U+10ffff]", 'U+%x' % e.code)
 
 
 # ____________________________________________________________
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -5,6 +5,8 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import interp_attrproperty
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
+from pypy.interpreter.unicodehelper import wcharpsize2utf8
+from pypy.interpreter.unicodehelper import wrap_unicode_out_of_range_error
 
 from rpython.rlib.clibffi import *
 from rpython.rlib.objectmodel import we_are_translated
@@ -598,10 +600,13 @@
     if address == 0:
         return space.w_None
     wcharp_addr = rffi.cast(rffi.CWCHARP, address)
-    if maxlength == -1:
-        s, lgt = rffi.wcharp2utf8(wcharp_addr)
-    else:
-        s, lgt = rffi.wcharp2utf8n(wcharp_addr, maxlength)
+    try:
+        if maxlength == -1:
+            s, lgt = rffi.wcharp2utf8(wcharp_addr)
+        else:
+            s, lgt = rffi.wcharp2utf8n(wcharp_addr, maxlength)
+    except rutf8.OutOfRange as e:
+        raise wrap_unicode_out_of_range_error(space, e)
     return space.newutf8(s, lgt)
 
 @unwrap_spec(address=r_uint, maxlength=int)
@@ -617,7 +622,7 @@
         return wcharp2unicode(space, address)
     elif maxlength < 0:
         maxlength = 0
-    s = rffi.wcharpsize2utf8(rffi.cast(rffi.CWCHARP, address), maxlength)
+    s = wcharpsize2utf8(space, rffi.cast(rffi.CWCHARP, address), maxlength)
     return space.newutf8(s, maxlength)
 
 @unwrap_spec(address=r_uint, newcontent='bufferstr', offset=int, size=int)
diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py
--- a/pypy/module/_rawffi/test/test__rawffi.py
+++ b/pypy/module/_rawffi/test/test__rawffi.py
@@ -1225,6 +1225,23 @@
         lib = _rawffi.CDLL(self.lib_name)
         assert lib.name == self.lib_name
 
+    def test_wcharp2rawunicode(self):
+        import _rawffi
+        A = _rawffi.Array('i')
+        arg = A(1)
+        arg[0] = 0x1234
+        u = _rawffi.wcharp2rawunicode(arg.itemaddress(0))
+        assert u == u'\u1234'
+        u = _rawffi.wcharp2rawunicode(arg.itemaddress(0), 1)
+        assert u == u'\u1234'
+        arg[0] = -1
+        raises(ValueError, _rawffi.wcharp2rawunicode, arg.itemaddress(0))
+        raises(ValueError, _rawffi.wcharp2rawunicode, arg.itemaddress(0), 1)
+        arg[0] = 0x110000
+        raises(ValueError, _rawffi.wcharp2rawunicode, arg.itemaddress(0))
+        raises(ValueError, _rawffi.wcharp2rawunicode, arg.itemaddress(0), 1)
+        arg.free()
+
 
 class AppTestAutoFree:
     spaceconfig = dict(usemodules=['_rawffi', 'struct'])
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -14,6 +14,7 @@
     interp2app, interpindirect2app, unwrap_spec)
 from pypy.interpreter.typedef import (
     GetSetProperty, TypeDef, make_weakref_descr)
+from pypy.interpreter.unicodehelper import wcharpsize2utf8
 
 
 @unwrap_spec(typecode='text')
@@ -509,7 +510,8 @@
             if s < 0:
                 s = 0
             buf = rffi.cast(UNICODE_ARRAY, self._buffer_as_unsigned())
-            return space.newutf8(rffi.wcharpsize2utf8(buf, s), s)
+            utf8 = wcharpsize2utf8(space, buf, s)
+            return space.newutf8(utf8, s)
         else:
             raise oefmt(space.w_ValueError,
                         "tounicode() may only be called on type 'u' arrays")
@@ -767,8 +769,16 @@
         if self.len == 0:
             return space.newtext("array('%s')" % self.typecode)
         elif self.typecode == "u":
-            r = space.repr(self.descr_tounicode(space))
-            s = "array('%s', %s)" % (self.typecode, space.text_w(r))
+            try:
+                w_unicode = self.descr_tounicode(space)
+            except OperationError as e:
+                if not e.match(space, space.w_ValueError):
+                    raise
+                w_exc_value = e.get_w_value(space)
+                r = "<%s>" % (space.text_w(w_exc_value),)
+            else:
+                r = space.text_w(space.repr(w_unicode))
+            s = "array('%s', %s)" % (self.typecode, r)
             return space.newtext(s)
         else:
             r = space.repr(self.descr_tolist(space))
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -893,10 +893,16 @@
         assert repr(mya('i', (1, 2, 3))) == "array('i', [1, 2, 3])"
 
     def test_unicode_outofrange(self):
-        a = self.array('u', u'\x01\u263a\x00\ufeff')
-        b = self.array('u', u'\x01\u263a\x00\ufeff')
+        input_unicode = u'\x01\u263a\x00\ufeff'
+        a = self.array('u', input_unicode)
+        b = self.array('u', input_unicode)
         b.byteswap()
         raises(ValueError, "a != b")
+        assert str(a) == "array('u', %r)" % (input_unicode,)
+        assert str(b) == ("array('u', )")
+        assert a.tounicode() == input_unicode
+        raises(ValueError, b.tounicode)   # doesn't work
 
     def test_weakref(self):
         import weakref
diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py
--- a/pypy/module/gc/referents.py
+++ b/pypy/module/gc/referents.py
@@ -20,7 +20,7 @@
     # inherits from W_Root for internal reasons.  Such instances don't
     # have a typedef at all (or have a null typedef after translation).
     if not we_are_translated():
-        if not hasattr(w_obj, 'typedef'):
+        if getattr(w_obj, 'typedef', None) is None:
             return None
     else:
         if w_obj is None or not w_obj.typedef:
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -1023,7 +1023,7 @@
 def wcharpsize2utf8(w, size):
     """ Helper to convert WCHARP pointer to utf8 in one go.
     Equivalent to wcharpsize2unicode().encode("utf8")
-    Raises ValueError if characters are outside range(0x110000)!
+    Raises rutf8.OutOfRange if characters are outside range(0x110000)!
     """
     from rpython.rlib import rutf8
 
@@ -1033,6 +1033,9 @@
     return s.build()
 
 def wcharp2utf8(w):
+    """
+    Raises rutf8.OutOfRange if characters are outside range(0x110000)!
+    """
     from rpython.rlib import rutf8
 
     s = rutf8.Utf8StringBuilder()
@@ -1043,6 +1046,9 @@
     return s.build(), i
 
 def wcharp2utf8n(w, maxlen):
+    """
+    Raises rutf8.OutOfRange if characters are outside range(0x110000)!
+    """
     from rpython.rlib import rutf8
 
     s = rutf8.Utf8StringBuilder(maxlen)

From pypy.commits at gmail.com  Sun Mar 31 08:22:06 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 31 Mar 2019 05:22:06 -0700 (PDT)
Subject: [pypy-commit] pypy default: preserve order on extra effects (sets
 are not ordered on cpython2)
Message-ID: <5ca0b0ee.1c69fb81.f8e9f.b857@mx.google.com>

Author: Matti Picus 
Branch: 
Changeset: r96388:a93dfb333afe
Date: 2019-03-31 15:15 +0300
http://bitbucket.org/pypy/pypy/changeset/a93dfb333afe/

Log:	preserve order on extra effects (sets are not ordered on cpython2)

diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py
--- a/rpython/jit/codewriter/effectinfo.py
+++ b/rpython/jit/codewriter/effectinfo.py
@@ -326,14 +326,17 @@
         # a read or a write to an interiorfield, inside an array of
         # structs, is additionally recorded as a read or write of
         # the array itself
-        extraef = set()
+        extraef = list()
         for tup in effects:
             if tup[0] == "interiorfield" or tup[0] == "readinteriorfield":
                 T = deref(tup[1])
                 if isinstance(T, lltype.Array) and consider_array(T):
-                    extraef.add((tup[0].replace("interiorfield", "array"),
-                                 tup[1]))
-        effects |= extraef
+                    val = (tup[0].replace("interiorfield", "array"),
+                                 tup[1])
+                    if val not in effects:
+                        extraef.append(val)
+        # preserve order in the added effects issue bitbucket #2984
+        effects = tuple(effects) + tuple(extraef)
 
         for tup in effects:
             if tup[0] == "struct":

From pypy.commits at gmail.com  Sun Mar 31 08:22:09 2019
From: pypy.commits at gmail.com (mattip)
Date: Sun, 31 Mar 2019 05:22:09 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge default into py3.6
Message-ID: <5ca0b0f1.1c69fb81.4f15.35c5@mx.google.com>

Author: Matti Picus 
Branch: py3.6
Changeset: r96389:4547d8edb215
Date: 2019-03-31 15:18 +0300
http://bitbucket.org/pypy/pypy/changeset/4547d8edb215/

Log:	merge default into py3.6

diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1597,6 +1597,17 @@
         w_res = self.run_and_check_stacksize(source)
         assert self.space.unwrap(w_res) == range(200)
 
+    def test_list_unpacking(self):
+        space = self.space
+        source = "[" + ",".join(['b%d' % i for i in range(200)]) + "] = a\n"
+        code = compile_with_astcompiler(source, 'exec', space)
+        assert code.co_stacksize == 200   # xxx remains big
+        w_dict = space.newdict()
+        space.setitem(w_dict, space.newtext("a"), space.wrap(range(42, 242)))
+        code.exec_code(space, w_dict, w_dict)
+        assert space.unwrap(space.getitem(w_dict, space.newtext("b0"))) == 42
+        assert space.unwrap(space.getitem(w_dict, space.newtext("b199"))) == 241
+
     def test_set(self):
         source = "{" + ",".join([str(i) for i in range(200)]) + "}\n"
         w_res = self.run_and_check_stacksize(source)
diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py
--- a/pypy/objspace/std/celldict.py
+++ b/pypy/objspace/std/celldict.py
@@ -11,17 +11,12 @@
     DictStrategy, ObjectDictStrategy, _never_equal_to_string,
     create_iterator_classes)
 from pypy.objspace.std.typeobject import (
-    MutableCell, IntMutableCell, ObjectMutableCell, write_cell)
+    MutableCell, IntMutableCell, ObjectMutableCell, write_cell, unwrap_cell)
 
 
 class VersionTag(object):
     pass
 
-def unwrap_cell(space, w_value):
-    if isinstance(w_value, MutableCell):
-        return w_value.unwrap_cell(space)
-    return w_value
-
 
 def _wrapkey(space, key):
     return space.newtext(key)
diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py
--- a/rpython/jit/codewriter/effectinfo.py
+++ b/rpython/jit/codewriter/effectinfo.py
@@ -326,14 +326,17 @@
         # a read or a write to an interiorfield, inside an array of
         # structs, is additionally recorded as a read or write of
         # the array itself
-        extraef = set()
+        extraef = list()
         for tup in effects:
             if tup[0] == "interiorfield" or tup[0] == "readinteriorfield":
                 T = deref(tup[1])
                 if isinstance(T, lltype.Array) and consider_array(T):
-                    extraef.add((tup[0].replace("interiorfield", "array"),
-                                 tup[1]))
-        effects |= extraef
+                    val = (tup[0].replace("interiorfield", "array"),
+                                 tup[1])
+                    if val not in effects:
+                        extraef.append(val)
+        # preserve order in the added effects issue bitbucket #2984
+        effects = tuple(effects) + tuple(extraef)
 
         for tup in effects:
             if tup[0] == "struct":
diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -264,6 +264,7 @@
             opnum == rop.SETFIELD_RAW or
             opnum == rop.SETARRAYITEM_RAW or
             opnum == rop.SETINTERIORFIELD_RAW or
+            opnum == rop.RECORD_EXACT_CLASS or
             opnum == rop.RAW_STORE or
             opnum == rop.ASSERT_NOT_NONE):
             return
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7210,6 +7210,7 @@
         [p0]
         p1 = getfield_gc_r(p0, descr=nextdescr)
         record_exact_class(p1, ConstClass(node_vtable))
+        guard_nonnull(p1) []
         guard_class(p1, ConstClass(node_vtable)) []
         jump(p1)
         """
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -287,8 +287,19 @@
         from rpython.rtyper.lltypesystem import llmemory
         if self.metainterp.heapcache.is_class_known(box):
             return
-        self.execute(rop.RECORD_EXACT_CLASS, box, clsbox)
-        self.metainterp.heapcache.class_now_known(box)
+        if isinstance(clsbox, Const):
+            self.execute(rop.RECORD_EXACT_CLASS, box, clsbox)
+            self.metainterp.heapcache.class_now_known(box)
+            self.metainterp.heapcache.nullity_now_known(box)
+        elif have_debug_prints():
+            if len(self.metainterp.framestack) >= 2:
+                # caller of ll_record_exact_class
+                name = self.metainterp.framestack[-2].jitcode.name
+            else:
+                name = self.jitcode.name
+            loc = self.metainterp.jitdriver_sd.warmstate.get_location_str(self.greenkey)
+            debug_print("record_exact_class with non-constant second argument, ignored",
+                    name, loc)
 
     @arguments("box")
     def _opimpl_any_return(self, box):
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -109,6 +109,35 @@
         self.check_trace_count(1)
         self.check_simple_loop(int_mul=1)
 
+    def test_rutf8(self):
+        from rpython.rlib import rutf8, jit
+        class U(object):
+            def __init__(self, u, l):
+                self.u = u
+                self.l = l
+                self._index_storage = rutf8.null_storage()
+
+            def _get_index_storage(self):
+                return jit.conditional_call_elidable(self._index_storage,
+                            U._compute_index_storage, self)
+
+            def _compute_index_storage(self):
+                storage = rutf8.create_utf8_index_storage(self.u, self.l)
+                self._index_storage = storage
+                return storage
+
+        def m(a):
+            return f(a)
+        def f(a):
+            x = str(a)
+            u = U(x, len(x))
+            st = u._get_index_storage()
+            return rutf8.codepoint_index_at_byte_position(
+                u.u, st, 1)
+
+        self.interp_operations(m, [123232])
+
+
     def test_loop_variant_mul_ovf(self):
         myjitdriver = JitDriver(greens = [], reds = ['y', 'res', 'x'])
         def f(x, y):
@@ -3960,6 +3989,55 @@
         # here it works again
         self.check_operations_history(guard_class=0, record_exact_class=1)
 
+    def test_record_exact_class_nonconst(self):
+        class Base(object):
+            def f(self):
+                raise NotImplementedError
+            def g(self):
+                raise NotImplementedError
+        class A(Base):
+            def f(self):
+                return self.a
+            def g(self):
+                return self.a + 1
+        class B(Base):
+            def f(self):
+                return self.b
+            def g(self):
+                return self.b + 1
+        class C(B):
+            def f(self):
+                self.c += 1
+                return self.c
+            def g(self):
+                return self.c + 1
+        @dont_look_inside
+        def make(x):
+            if x > 0:
+                a = A()
+                a.a = x + 1
+            elif x < 0:
+                a = B()
+                a.b = -x
+            else:
+                a = C()
+                a.c = 10
+            return a, type(a)
+        def f(x):
+            a, cls = make(x)
+            record_exact_class(a, cls)
+            if x > 0:
+                z = a.f()
+            elif x < 0:
+                z = a.f()
+            else:
+                z = a.f()
+            return z + a.g()
+        res1 = f(6)
+        res2 = self.interp_operations(f, [6])
+        assert res1 == res2
+        self.check_operations_history(guard_class=1, record_exact_class=0)
+
     def test_generator(self):
         def g(n):
             yield n+1
diff --git a/rpython/jit/metainterp/test/test_tracingopts.py b/rpython/jit/metainterp/test/test_tracingopts.py
--- a/rpython/jit/metainterp/test/test_tracingopts.py
+++ b/rpython/jit/metainterp/test/test_tracingopts.py
@@ -743,3 +743,30 @@
         res = self.interp_operations(fn, [0])
         assert res == 0
         self.check_operations_history(setfield_gc=0)
+
+    def test_record_known_class_does_not_invalidate(self):
+        class A:
+            pass
+        class B(A):
+            pass
+        class C(object):
+            _immutable_fields_ = ['x?']
+        c = C()
+        c.x = 5
+        c.b = A()
+        c.b.x = 14
+        def fn(n):
+            if n == 99:
+                c.x = 12
+                c.b = B()
+                c.b.x = 12
+                return 15
+            b = c.b
+            x = b.x
+            jit.record_exact_class(c.b, A)
+            y = b.x
+            return x + y
+        res = self.interp_operations(fn, [1])
+        assert res == 2 * 14
+        self.check_operations_history(getfield_gc_i=1)
+
diff --git a/rpython/jit/metainterp/test/test_virtualizable.py b/rpython/jit/metainterp/test/test_virtualizable.py
--- a/rpython/jit/metainterp/test/test_virtualizable.py
+++ b/rpython/jit/metainterp/test/test_virtualizable.py
@@ -542,6 +542,41 @@
         self.check_resops(getfield_gc_r=1, getarrayitem_gc_i=4,
                           getfield_gc_i=1)
 
+    @py.test.mark.xfail
+    def test_virtualizable_with_array_huge(self):
+        myjitdriver = JitDriver(greens = [], reds = ['n', 'x', 'frame'],
+                                virtualizables = ['frame'])
+
+        class Frame(object):
+            _virtualizable_ = ['l[*]', 's']
+
+            def __init__(self, l, s):
+                self.l = l
+                self.s = s
+
+        def f(n, a):
+            frame = Frame([a] * 1024, 0) # make a huge frame
+            x = 0
+            while n > 0:
+                myjitdriver.can_enter_jit(frame=frame, n=n, x=x)
+                myjitdriver.jit_merge_point(frame=frame, n=n, x=x)
+                frame.s = promote(frame.s)
+                n -= 1
+                s = frame.s
+                assert s >= 0
+                x += frame.l[s]
+                frame.s += 1
+                s = frame.s
+                assert s >= 0
+                x += frame.l[s]
+                x += len(frame.l)
+                frame.s -= 1
+            return x
+
+        res = self.meta_interp(f, [50, 1], listops=True)
+        # should stop giving up to compile, eventually
+        assert get_stats().aborted_count < 6
+
     def test_subclass_of_virtualizable(self):
         myjitdriver = JitDriver(greens = [], reds = ['frame'],
                                 virtualizables = ['frame'])
diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -22,7 +22,7 @@
     prebuilt_gc_objects_are_static_roots = True
     can_usually_pin_objects = False
     object_minimal_size = 0
-    gcflag_extra = 0   # or a real GC flag that is always 0 when not collecting
+    gcflag_extra = 0   # or a dedicated GC flag that the GC initializes to 0
     _totalroots_rpy = 0   # for inspector.py
 
     def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE,
diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -261,7 +261,7 @@
                 132: 13, 145: 41, 158: 13, 161: 2, 164: 11, 167: 13, 183: 17,
                 188: 8, 189: 8, 190: 8, 191: 8, 192: 8, 193: 8, 194: 8,
                 195: 8, 196: 8, 197: 8, 198: 8, 199: 8, 200: 8, 201: 8,
-                202: 8, 206: 2, 215: 11, 232: 32, 267: 20, 1816: 12, 1225: 111,
+                202: 8, 206: 2, 215: 11, 232: 32, 267: 20, 1816: 12,
                 }
         return errors, errno.EINVAL
 
diff --git a/rpython/tool/jitlogparser/test/test_modulefinder.py b/rpython/tool/jitlogparser/test/test_modulefinder.py
deleted file mode 100644
--- a/rpython/tool/jitlogparser/test/test_modulefinder.py
+++ /dev/null
@@ -1,22 +0,0 @@
-import py
-from rpython.tool.jitlogparser.module_finder import gather_all_code_objs
-import re, sys
-
-def setup_module(mod):
-    if sys.version_info[:2] != (2, 6):
-        py.test.skip("Specific python 2.6 tests")
-
-def test_gather_code_py():
-    py.test.skip("XXX broken, fix me")
-    fname = re.__file__
-    codes = gather_all_code_objs(fname)
-    assert len(codes) == 21
-    assert sorted(codes.keys()) == [102, 134, 139, 144, 153, 164, 169, 181, 188, 192, 197, 206, 229, 251, 266, 271, 277, 285, 293, 294, 308]
-
-def test_load_code():
-    py.test.skip("XXX broken, fix me")
-    fname = re.__file__
-    code = gather_all_code_objs(fname)[144]
-    assert code.co_name == 'sub'
-    assert code.co_filename == '/usr/lib/python2.6/re.py'
-    assert code.co_firstlineno == 144

From pypy.commits at gmail.com  Sun Mar 31 11:46:40 2019
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 31 Mar 2019 08:46:40 -0700 (PDT)
Subject: [pypy-commit] pypy default: Clean up test
Message-ID: <5ca0e0e0.1c69fb81.6c536.4369@mx.google.com>

Author: Ronan Lamy 
Branch: 
Changeset: r96390:3c18b70c56bd
Date: 2019-03-31 16:45 +0100
http://bitbucket.org/pypy/pypy/changeset/3c18b70c56bd/

Log:	Clean up test

diff --git a/rpython/rlib/rjitlog/test/test_jitlog.py b/rpython/rlib/rjitlog/test/test_jitlog.py
--- a/rpython/rlib/rjitlog/test/test_jitlog.py
+++ b/rpython/rlib/rjitlog/test/test_jitlog.py
@@ -1,4 +1,4 @@
-import py
+import pytest
 import sys
 from rpython.jit.tool.oparser import pure_parse
 from rpython.jit.metainterp.optimizeopt.util import equaloplists
@@ -27,37 +27,38 @@
     assert len(greenkey_list) == 0
     return '/home/pypy/jit.py', 0, 'enclosed', 99, 'DEL'
 
+class FakeJitDriver(object):
+    class warmstate(object):
+        get_location_types = [jl.MP_FILENAME,jl.MP_INT,jl.MP_SCOPE, jl.MP_INT, jl.MP_OPCODE]
+        @staticmethod
+        def get_location(greenkey_list):
+            return [jl.wrap(jl.MP_FILENAME[0],'s','/home/pypy/jit.py'),
+                    jl.wrap(jl.MP_INT[0], 'i', 0),
+                    jl.wrap(jl.MP_SCOPE[0], 's', 'enclosed'),
+                    jl.wrap(jl.MP_INT[0], 'i', 99),
+                    jl.wrap(jl.MP_OPCODE[0], 's', 'DEL')
+                    ]
+
+
+class FakeMetaInterpSd:
+    cpu = AbstractCPU()
+    cpu.ts = None
+    jitdrivers_sd = [FakeJitDriver()]
+    def get_name_from_address(self, addr):
+        return 'Name'
+
+ at pytest.fixture
+def metainterp_sd():
+    return FakeMetaInterpSd()
+
 class TestLogger(object):
-
-    def make_metainterp_sd(self):
-        class FakeJitDriver(object):
-            class warmstate(object):
-                get_location_types = [jl.MP_FILENAME,jl.MP_INT,jl.MP_SCOPE, jl.MP_INT, jl.MP_OPCODE]
-                @staticmethod
-                def get_location(greenkey_list):
-                    return [jl.wrap(jl.MP_FILENAME[0],'s','/home/pypy/jit.py'),
-                            jl.wrap(jl.MP_INT[0], 'i', 0),
-                            jl.wrap(jl.MP_SCOPE[0], 's', 'enclosed'),
-                            jl.wrap(jl.MP_INT[0], 'i', 99),
-                            jl.wrap(jl.MP_OPCODE[0], 's', 'DEL')
-                           ]
-
-
-        class FakeMetaInterpSd:
-            cpu = AbstractCPU()
-            cpu.ts = None
-            jitdrivers_sd = [FakeJitDriver()]
-            def get_name_from_address(self, addr):
-                return 'Name'
-        return FakeMetaInterpSd()
-
-    def test_debug_merge_point(self, tmpdir):
+    def test_debug_merge_point(self, tmpdir, metainterp_sd):
         logger = jl.JitLogger()
         file = tmpdir.join('binary_file')
         file.ensure()
         fd = file.open('wb')
         jl.jitlog_init(fd.fileno())
-        logger.start_new_trace(self.make_metainterp_sd(), jd_name='jdname')
+        logger.start_new_trace(metainterp_sd, jd_name='jdname')
         log_trace = logger.log_trace(jl.MARK_TRACE, None, None)
         op = ResOperation(rop.DEBUG_MERGE_POINT, [ConstInt(0), ConstInt(0), ConstInt(0)])
         log_trace.write([], [op])
@@ -115,10 +116,10 @@
         assert jl.commonprefix("/hello/world","/path/to") == "/"
         assert jl.commonprefix("pyramid","python") == "py"
         assert jl.commonprefix("0"*100,"0"*100) == "0"*100
-        with py.test.raises(AssertionError):
+        with pytest.raises(AssertionError):
             jl.commonprefix(None,None)
 
-    def test_redirect_assembler(self, tmpdir):
+    def test_redirect_assembler(self, tmpdir, metainterp_sd):
         looptoken = FakeCallAssemblerLoopToken(0x0)
         newlooptoken = FakeCallAssemblerLoopToken(0x1234)
         #
@@ -127,7 +128,7 @@
         file.ensure()
         fd = file.open('wb')
         jl.jitlog_init(fd.fileno())
-        logger.start_new_trace(self.make_metainterp_sd(), jd_name='jdname')
+        logger.start_new_trace(metainterp_sd, jd_name='jdname')
         log_trace = logger.log_trace(jl.MARK_TRACE, None, None)
         op = ResOperation(rop.CALL_ASSEMBLER_I, [], descr=looptoken)
         log_trace.write([], [op])
@@ -145,4 +146,3 @@
               jl.encode_le_addr(new_id_looptoken) + \
               jl.encode_le_addr(newlooptoken._ll_function_addr)
         assert binary.endswith(end)
-