From pypy.commits at gmail.com Mon Jan 1 16:10:54 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 01 Jan 2018 13:10:54 -0800 (PST) Subject: [pypy-commit] pypy default: enhance rwinreg to more easily allow python2 to 3, 'fix' test to properly skip Message-ID: <5a4aa3de.e280df0a.2c974.6c0e@mx.google.com> Author: Matti Picus Branch: Changeset: r93606:c5b147d52043 Date: 2017-12-30 23:00 +0200 http://bitbucket.org/pypy/pypy/changeset/c5b147d52043/ Log: enhance rwinreg to more easily allow python2 to 3, 'fix' test to properly skip diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py --- a/pypy/module/_winreg/test/test_winreg.py +++ b/pypy/module/_winreg/test/test_winreg.py @@ -12,11 +12,11 @@ priv_flags = win32security.TOKEN_ADJUST_PRIVILEGES | win32security.TOKEN_QUERY hToken = win32security.OpenProcessToken (win32api.GetCurrentProcess (), priv_flags) privilege_id = win32security.LookupPrivilegeValue (None, "SeBackupPrivilege") - win32security.AdjustTokenPrivileges (hToken, 0, [(privilege_id, win32security.SE_PRIVILEGE_ENABLED)]) + ret = win32security.AdjustTokenPrivileges (hToken, 0, [(privilege_id, win32security.SE_PRIVILEGE_ENABLED)]) except: canSaveKey = False else: - canSaveKey = True + canSaveKey = len(ret) > 0 class AppTestHKey: spaceconfig = dict(usemodules=('_winreg',)) @@ -174,6 +174,7 @@ raises(EnvironmentError, EnumKey, key, 1) def test_delete(self): + # must be run after test_SetValueEx from _winreg import OpenKey, KEY_ALL_ACCESS, DeleteValue, DeleteKey key = OpenKey(self.root_key, self.test_key_name, 0, KEY_ALL_ACCESS) sub_key = OpenKey(key, "sub_key", 0, KEY_ALL_ACCESS) @@ -189,6 +190,7 @@ h.Close() def test_savekey(self): + # must be run after test_SetValueEx if not self.canSaveKey: skip("CPython needs win32api to set the SeBackupPrivilege security privilege") from _winreg import OpenKey, KEY_ALL_ACCESS, SaveKey diff --git a/rpython/rlib/rwinreg.py b/rpython/rlib/rwinreg.py --- a/rpython/rlib/rwinreg.py +++ b/rpython/rlib/rwinreg.py @@ -47,71 +47,72 @@ HKEY = rwin32.HANDLE PHKEY = rffi.CArrayPtr(HKEY) REGSAM = rwin32.DWORD +suffix = 'A' RegSetValue = external( - 'RegSetValueA', + 'RegSetValue' + suffix, [HKEY, rffi.CCHARP, rwin32.DWORD, rffi.CCHARP, rwin32.DWORD], rffi.LONG) RegSetValueEx = external( - 'RegSetValueExA', + 'RegSetValueEx' + suffix, [HKEY, rffi.CCHARP, rwin32.DWORD, rwin32.DWORD, rffi.CCHARP, rwin32.DWORD], rffi.LONG) RegQueryValue = external( - 'RegQueryValueA', + 'RegQueryValue' + suffix, [HKEY, rffi.CCHARP, rffi.CCHARP, rwin32.PLONG], rffi.LONG) RegQueryValueEx = external( - 'RegQueryValueExA', + 'RegQueryValueEx' + suffix, [HKEY, rffi.CCHARP, rwin32.LPDWORD, rwin32.LPDWORD, rffi.CCHARP, rwin32.LPDWORD], rffi.LONG) RegCreateKey = external( - 'RegCreateKeyA', + 'RegCreateKey' + suffix, [HKEY, rffi.CCHARP, PHKEY], rffi.LONG) RegCreateKeyEx = external( - 'RegCreateKeyExA', + 'RegCreateKeyEx' + suffix, [HKEY, rffi.CCHARP, rwin32.DWORD, rffi.CCHARP, rwin32.DWORD, REGSAM, rffi.VOIDP, PHKEY, rwin32.LPDWORD], rffi.LONG) RegDeleteValue = external( - 'RegDeleteValueA', + 'RegDeleteValue' + suffix, [HKEY, rffi.CCHARP], rffi.LONG) RegDeleteKey = external( - 'RegDeleteKeyA', + 'RegDeleteKey' + suffix, [HKEY, rffi.CCHARP], rffi.LONG) RegOpenKeyEx = external( - 'RegOpenKeyExA', + 'RegOpenKeyEx' + suffix, [HKEY, rffi.CCHARP, rwin32.DWORD, REGSAM, PHKEY], rffi.LONG) RegEnumValue = external( - 'RegEnumValueA', + 'RegEnumValue' + suffix, [HKEY, rwin32.DWORD, rffi.CCHARP, rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, rffi.CCHARP, rwin32.LPDWORD], rffi.LONG) RegEnumKeyEx = external( - 'RegEnumKeyExA', + 'RegEnumKeyEx' + suffix, [HKEY, rwin32.DWORD, rffi.CCHARP, rwin32.LPDWORD, rwin32.LPDWORD, rffi.CCHARP, rwin32.LPDWORD, rwin32.PFILETIME], rffi.LONG) RegQueryInfoKey = external( - 'RegQueryInfoKeyA', + 'RegQueryInfoKey' + suffix, [HKEY, rffi.CCHARP, rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, @@ -129,17 +130,17 @@ rffi.LONG) RegLoadKey = external( - 'RegLoadKeyA', + 'RegLoadKey' + suffix, [HKEY, rffi.CCHARP, rffi.CCHARP], rffi.LONG) RegSaveKey = external( - 'RegSaveKeyA', + 'RegSaveKey' + suffix, [HKEY, rffi.CCHARP, rffi.VOIDP], rffi.LONG) RegConnectRegistry = external( - 'RegConnectRegistryA', + 'RegConnectRegistry' + suffix, [rffi.CCHARP, HKEY, PHKEY], rffi.LONG) From pypy.commits at gmail.com Mon Jan 1 16:10:56 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 01 Jan 2018 13:10:56 -0800 (PST) Subject: [pypy-commit] pypy py3.5: enhance rwinreg to more easily allow python2 to 3, 'fix' test to properly skip Message-ID: <5a4aa3e0.e5b2df0a.10835.d2ac@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93607:6c36fd9c5c97 Date: 2017-12-30 23:00 +0200 http://bitbucket.org/pypy/pypy/changeset/6c36fd9c5c97/ Log: enhance rwinreg to more easily allow python2 to 3, 'fix' test to properly skip (grafted from c5b147d52043e8eafbe5abbb435a7ad46c4f712f) diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py --- a/pypy/module/_winreg/test/test_winreg.py +++ b/pypy/module/_winreg/test/test_winreg.py @@ -12,11 +12,11 @@ priv_flags = win32security.TOKEN_ADJUST_PRIVILEGES | win32security.TOKEN_QUERY hToken = win32security.OpenProcessToken (win32api.GetCurrentProcess (), priv_flags) privilege_id = win32security.LookupPrivilegeValue (None, "SeBackupPrivilege") - win32security.AdjustTokenPrivileges (hToken, 0, [(privilege_id, win32security.SE_PRIVILEGE_ENABLED)]) + ret = win32security.AdjustTokenPrivileges (hToken, 0, [(privilege_id, win32security.SE_PRIVILEGE_ENABLED)]) except: canSaveKey = False else: - canSaveKey = True + canSaveKey = len(ret) > 0 class AppTestHKey: #spaceconfig = dict(usemodules=('_winreg',)) @@ -177,7 +177,8 @@ raises(EnvironmentError, EnumKey, key, 1) def test_delete(self): - from winreg import OpenKey, KEY_ALL_ACCESS, DeleteValue, DeleteKey + # must be run after test_SetValueEx + from _winreg import OpenKey, KEY_ALL_ACCESS, DeleteValue, DeleteKey key = OpenKey(self.root_key, self.test_key_name, 0, KEY_ALL_ACCESS) sub_key = OpenKey(key, "sub_key", 0, KEY_ALL_ACCESS) @@ -192,6 +193,7 @@ h.Close() def test_savekey(self): + # must be run after test_SetValueEx if not self.canSaveKey: skip("CPython needs win32api to set the SeBackupPrivilege security privilege") from winreg import OpenKey, KEY_ALL_ACCESS, SaveKey diff --git a/rpython/rlib/rwinreg.py b/rpython/rlib/rwinreg.py --- a/rpython/rlib/rwinreg.py +++ b/rpython/rlib/rwinreg.py @@ -47,71 +47,72 @@ HKEY = rwin32.HANDLE PHKEY = rffi.CArrayPtr(HKEY) REGSAM = rwin32.DWORD +suffix = 'A' RegSetValue = external( - 'RegSetValueA', + 'RegSetValue' + suffix, [HKEY, rffi.CCHARP, rwin32.DWORD, rffi.CCHARP, rwin32.DWORD], rffi.LONG) RegSetValueEx = external( - 'RegSetValueExA', + 'RegSetValueEx' + suffix, [HKEY, rffi.CCHARP, rwin32.DWORD, rwin32.DWORD, rffi.CCHARP, rwin32.DWORD], rffi.LONG) RegQueryValue = external( - 'RegQueryValueA', + 'RegQueryValue' + suffix, [HKEY, rffi.CCHARP, rffi.CCHARP, rwin32.PLONG], rffi.LONG) RegQueryValueEx = external( - 'RegQueryValueExA', + 'RegQueryValueEx' + suffix, [HKEY, rffi.CCHARP, rwin32.LPDWORD, rwin32.LPDWORD, rffi.CCHARP, rwin32.LPDWORD], rffi.LONG) RegCreateKey = external( - 'RegCreateKeyA', + 'RegCreateKey' + suffix, [HKEY, rffi.CCHARP, PHKEY], rffi.LONG) RegCreateKeyEx = external( - 'RegCreateKeyExA', + 'RegCreateKeyEx' + suffix, [HKEY, rffi.CCHARP, rwin32.DWORD, rffi.CCHARP, rwin32.DWORD, REGSAM, rffi.VOIDP, PHKEY, rwin32.LPDWORD], rffi.LONG) RegDeleteValue = external( - 'RegDeleteValueA', + 'RegDeleteValue' + suffix, [HKEY, rffi.CCHARP], rffi.LONG) RegDeleteKey = external( - 'RegDeleteKeyA', + 'RegDeleteKey' + suffix, [HKEY, rffi.CCHARP], rffi.LONG) RegOpenKeyEx = external( - 'RegOpenKeyExA', + 'RegOpenKeyEx' + suffix, [HKEY, rffi.CCHARP, rwin32.DWORD, REGSAM, PHKEY], rffi.LONG) RegEnumValue = external( - 'RegEnumValueA', + 'RegEnumValue' + suffix, [HKEY, rwin32.DWORD, rffi.CCHARP, rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, rffi.CCHARP, rwin32.LPDWORD], rffi.LONG) RegEnumKeyEx = external( - 'RegEnumKeyExA', + 'RegEnumKeyEx' + suffix, [HKEY, rwin32.DWORD, rffi.CCHARP, rwin32.LPDWORD, rwin32.LPDWORD, rffi.CCHARP, rwin32.LPDWORD, rwin32.PFILETIME], rffi.LONG) RegQueryInfoKey = external( - 'RegQueryInfoKeyA', + 'RegQueryInfoKey' + suffix, [HKEY, rffi.CCHARP, rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, @@ -129,17 +130,17 @@ rffi.LONG) RegLoadKey = external( - 'RegLoadKeyA', + 'RegLoadKey' + suffix, [HKEY, rffi.CCHARP, rffi.CCHARP], rffi.LONG) RegSaveKey = external( - 'RegSaveKeyA', + 'RegSaveKey' + suffix, [HKEY, rffi.CCHARP, rffi.VOIDP], rffi.LONG) RegConnectRegistry = external( - 'RegConnectRegistryA', + 'RegConnectRegistry' + suffix, [rffi.CCHARP, HKEY, PHKEY], rffi.LONG) From pypy.commits at gmail.com Mon Jan 1 16:10:52 2018 From: pypy.commits at gmail.com (rlamy) Date: Mon, 01 Jan 2018 13:10:52 -0800 (PST) Subject: [pypy-commit] pypy release-pypy3.5-v5.9.x: Fix err.filename when *xattr() functions raise an OSError Message-ID: <5a4aa3dc.e280df0a.2c974.6c0a@mx.google.com> Author: Ronan Lamy Branch: release-pypy3.5-v5.9.x Changeset: r93605:f1ac52de24a1 Date: 2017-12-29 20:06 +0100 http://bitbucket.org/pypy/pypy/changeset/f1ac52de24a1/ Log: Fix err.filename when *xattr() functions raise an OSError (grafted from 49d834e68cbe3efcdd90011deda203ac65571fa1) diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -2304,13 +2304,13 @@ try: result = rposix.fgetxattr(path.as_fd, attribute.as_bytes) except OSError as e: - raise wrap_oserror(space, e, path.as_bytes) + raise wrap_oserror2(space, e, path.w_path) else: try: result = rposix.getxattr(path.as_bytes, attribute.as_bytes, follow_symlinks=follow_symlinks) except OSError as e: - raise wrap_oserror(space, e, path.as_bytes) + raise wrap_oserror2(space, e, path.w_path) return space.newbytes(result) @unwrap_spec(path=path_or_fd(), attribute=path_or_fd(allow_fd=False), @@ -2333,13 +2333,13 @@ try: rposix.fsetxattr(path.as_fd, attribute.as_bytes, value, flags) except OSError as e: - raise wrap_oserror(space, e, path.as_bytes) + raise wrap_oserror2(space, e, path.w_path) else: try: rposix.setxattr(path.as_bytes, attribute.as_bytes, value, flags, follow_symlinks=follow_symlinks) except OSError as e: - raise wrap_oserror(space, e, path.as_bytes) + raise wrap_oserror2(space, e, path.w_path) @unwrap_spec(path=path_or_fd(), attribute=path_or_fd(allow_fd=False), @@ -2359,13 +2359,13 @@ try: rposix.fremovexattr(path.as_fd, attribute.as_bytes) except OSError as e: - raise wrap_oserror(space, e, path.as_bytes) + raise wrap_oserror2(space, e, path.w_path) else: try: rposix.removexattr(path.as_bytes, attribute.as_bytes, follow_symlinks=follow_symlinks) except OSError as e: - raise wrap_oserror(space, e, path.as_bytes) + raise wrap_oserror2(space, e, path.w_path) @unwrap_spec(path=path_or_fd(), follow_symlinks=bool) @@ -2386,12 +2386,12 @@ try: result = rposix.flistxattr(path.as_fd) except OSError as e: - raise wrap_oserror(space, e, eintr_retry=False) + raise wrap_oserror2(space, e, path.w_path) else: try: result = rposix.listxattr(path.as_bytes, follow_symlinks) except OSError as e: - raise wrap_oserror(space, e, path.as_bytes) + raise wrap_oserror2(space, e, path.w_path) return space.newlist([space.newfilename(attr) for attr in result]) From pypy.commits at gmail.com Mon Jan 1 16:10:58 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 01 Jan 2018 13:10:58 -0800 (PST) Subject: [pypy-commit] pypy py3-winreg: wip - test start to pass Message-ID: <5a4aa3e2.af96df0a.1da9b.4843@mx.google.com> Author: Matti Picus Branch: py3-winreg Changeset: r93608:be6cdcfdb592 Date: 2018-01-01 23:03 +0200 http://bitbucket.org/pypy/pypy/changeset/be6cdcfdb592/ Log: wip - test start to pass diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py --- a/pypy/interpreter/gateway.py +++ b/pypy/interpreter/gateway.py @@ -171,6 +171,9 @@ def visit_text0(self, el, app_sig): self.checked_space_method(el, app_sig) + def visit_unicode(self, el, app_sig): + self.checked_space_method(el, app_sig) + def visit_fsencode(self, el, app_sig): self.checked_space_method(el, app_sig) @@ -320,6 +323,9 @@ def visit_text0(self, typ): self.run_args.append("space.text0_w(%s)" % (self.scopenext(),)) + def visit_unicode(self, typ): + self.run_args.append("space.unicode_w(%s)" % (self.scopenext(),)) + def visit_fsencode(self, typ): self.run_args.append("space.fsencode_w(%s)" % (self.scopenext(),)) @@ -485,6 +491,9 @@ def visit_text(self, typ): self.unwrap.append("space.text_w(%s)" % (self.nextarg(),)) + def visit_unicode(self, typ): + self.unwrap.append("space.unicode_w(%s)" % (self.nextarg(),)) + def visit_text0(self, typ): self.unwrap.append("space.text0_w(%s)" % (self.nextarg(),)) diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py --- a/pypy/module/_winreg/interp_winreg.py +++ b/pypy/module/_winreg/interp_winreg.py @@ -8,10 +8,10 @@ from rpython.rlib.rarithmetic import r_uint, intmask def raiseWindowsError(space, errcode, context): - message = rwin32.FormatErrorW(errcode) + message = rwin32.FormatError(errcode) raise OperationError(space.w_WindowsError, space.newtuple([space.newint(errcode), - space.newunicode(message)])) + space.newtext(message)])) class W_HKEY(W_Root): def __init__(self, space, hkey): @@ -32,7 +32,7 @@ return space.newint(self.as_int()) def descr_repr(self, space): - return space.newtext("" % (self.as_int(),)) + return space.newunicode(u"" % (self.as_int(),)) def descr_int(self, space): return space.newint(self.as_int()) @@ -110,8 +110,8 @@ elif isinstance(w_hkey, W_HKEY): return w_hkey.hkey elif space.isinstance_w(w_hkey, space.w_int): - if space.is_true(space.lt(w_hkey, space.newint(0))): - return rffi.cast(rwinreg.HKEY, space.int_w(w_hkey)) + return rffi.cast(rwinreg.HKEY, space.int_w(w_hkey)) + elif space.isinstance_w(w_hkey, space.w_long): return rffi.cast(rwinreg.HKEY, space.uint_w(w_hkey)) else: raise oefmt(space.w_TypeError, "The object is not a PyHKEY object") @@ -149,7 +149,7 @@ if ret != 0: raiseWindowsError(space, ret, 'RegFlushKey') - at unwrap_spec(subkey="text", filename="text") + at unwrap_spec(subkey="unicode", filename="unicode") def LoadKey(space, w_hkey, subkey, filename): """LoadKey(key, sub_key, file_name) - Creates a subkey under the specified key and stores registration information from a specified file into that subkey. @@ -174,7 +174,7 @@ if ret != 0: raiseWindowsError(space, ret, 'RegLoadKey') - at unwrap_spec(filename="text") + at unwrap_spec(filename="unicode") def SaveKey(space, w_hkey, filename): """SaveKey(key, file_name) - Saves the specified key, and all its subkeys to the specified file. @@ -193,7 +193,7 @@ if ret != 0: raiseWindowsError(space, ret, 'RegSaveKey') - at unwrap_spec(typ=int, value="text") + at unwrap_spec(typ=int, value="unicode") def SetValue(space, w_hkey, w_subkey, typ, value): """SetValue(key, sub_key, type, value) - Associates a value with a specified key. @@ -215,14 +215,14 @@ if typ != rwinreg.REG_SZ: raise oefmt(space.w_ValueError, "Type must be winreg.REG_SZ") hkey = hkey_w(w_hkey, space) - if space.is_w(w_subkey, space.w_None): - subkey = None - else: - subkey = space.text_w(w_subkey) - with rffi.scoped_str2charp(value) as dataptr: - ret = rwinreg.RegSetValue(hkey, subkey, rwinreg.REG_SZ, dataptr, len(value)) - if ret != 0: - raiseWindowsError(space, ret, 'RegSetValue') + with rffi.scoped_unicode2wcharp(space.unicode_w(w_subkey)) as subkey: + c_subkey = rffi.cast(rffi.CCHARP, subkey) + with rffi.scoped_unicode2wcharp(value) as dataptr: + c_dataptr = rffi.cast(rffi.CCHARP, dataptr) + ret = rwinreg.RegSetValue(hkey, c_subkey, rwinreg.REG_SZ, + c_dataptr, len(value)) + if ret != 0: + raiseWindowsError(space, ret, 'RegSetValue') def QueryValue(space, w_hkey, w_subkey): """string = QueryValue(key, sub_key) - retrieves the unnamed value for a key. @@ -239,28 +239,32 @@ if space.is_w(w_subkey, space.w_None): subkey = None else: - subkey = space.text_w(w_subkey) - with lltype.scoped_alloc(rwin32.PLONG.TO, 1) as bufsize_p: - ret = rwinreg.RegQueryValue(hkey, subkey, None, bufsize_p) - bufSize = intmask(bufsize_p[0]) - if ret == rwinreg.ERROR_MORE_DATA: - bufSize = 256 - elif ret != 0: - raiseWindowsError(space, ret, 'RegQueryValue') + subkey = space.unicode_w(w_subkey) + with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: + c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) + with lltype.scoped_alloc(rwin32.PLONG.TO, 1) as bufsize_p: + ret = rwinreg.RegQueryValue(hkey, c_subkey, None, bufsize_p) + bufSize = intmask(bufsize_p[0]) + if ret == rwinreg.ERROR_MORE_DATA: + bufSize = 256 + elif ret != 0: + raiseWindowsError(space, ret, 'RegQueryValue') - while True: - with lltype.scoped_alloc(rffi.CCHARP.TO, bufSize) as buf: - ret = rwinreg.RegQueryValue(hkey, subkey, buf, bufsize_p) - if ret == rwinreg.ERROR_MORE_DATA: - # Resize and retry - bufSize *= 2 - bufsize_p[0] = bufSize - continue + while True: + with lltype.scoped_alloc(rffi.CCHARP.TO, bufSize) as buf: + ret = rwinreg.RegQueryValue(hkey, c_subkey, buf, bufsize_p) + if ret == rwinreg.ERROR_MORE_DATA: + print 'bufSize was %d, too small' % bufSize + # Resize and retry + bufSize *= 2 + bufsize_p[0] = bufSize + continue - if ret != 0: - raiseWindowsError(space, ret, 'RegQueryValue') - length = intmask(bufsize_p[0] - 1) - return space.newtext(rffi.charp2strn(buf, length)) + if ret != 0: + raiseWindowsError(space, ret, 'RegQueryValue') + length = intmask(bufsize_p[0] - 1) / 2 + wide_buf = rffi.cast(rffi.CWCHARP, buf) + return space.newunicode(rffi.wcharp2unicoden(wide_buf, length)) def convert_to_regdata(space, w_value, typ): buf = None @@ -282,10 +286,8 @@ buf = lltype.malloc(rffi.CCHARP.TO, buflen, flavor='raw') buf[0] = '\0' else: - if space.isinstance_w(w_value, space.w_unicode): - w_value = space.call_method(w_value, 'encode', - space.newtext('mbcs')) - buf = rffi.str2charp(space.text_w(w_value)) + buf = rffi.unicode2wcharp(space.unicode_w(w_value)) + buf = rffi.cast(rffi.CCHARP, buf) buflen = space.len_w(w_value) + 1 elif typ == rwinreg.REG_MULTI_SZ: @@ -302,10 +304,7 @@ while True: try: w_item = space.next(w_iter) - if space.isinstance_w(w_item, space.w_unicode): - w_item = space.call_method(w_item, 'encode', - space.newtext('mbcs')) - item = space.bytes_w(w_item) + item = space.unicode_w(w_item) strings.append(item) buflen += len(item) + 1 except OperationError as e: @@ -318,8 +317,10 @@ # Now copy data buflen = 0 for string in strings: - for i in range(len(string)): - buf[buflen + i] = string[i] + with rffi.scoped_unicode2wcharp(string) as wchr: + c_str = rffi.cast(rffi.CCHARP, wchr) + for i in range(len(string)): + buf[buflen + i] = wchr[i] buflen += len(string) + 1 buf[buflen - 1] = '\0' buflen += 1 @@ -360,12 +361,12 @@ s = "" else: # may or may not have a trailing NULL in the buffer. - buf = rffi.cast(rffi.CCHARP, buf) + buf = rffi.cast(rffi.CWCHARP, buf) if buf[buflen - 1] == '\x00': buflen -= 1 - s = rffi.charp2strn(buf, buflen) + s = rffi.wcharp2unicoden(buf, buflen) w_s = space.newbytes(s) - return space.call_method(w_s, 'decode', space.newtext('mbcs')) + return w_s elif typ == rwinreg.REG_MULTI_SZ: if not buflen: @@ -380,14 +381,14 @@ if len(s) == 0: break s = ''.join(s) - l.append(space.newtext(s)) + l.append(space.newunicode(s)) i += 1 return space.newlist(l) else: # REG_BINARY and all other types return space.newbytes(rffi.charpsize2str(buf, buflen)) - at unwrap_spec(value_name="text", typ=int) + at unwrap_spec(value_name="unicode", typ=int) def SetValueEx(space, w_hkey, value_name, w_reserved, typ, w_value): """SetValueEx(key, value_name, reserved, type, value) - Stores data in the value field of an open registry key. @@ -422,7 +423,9 @@ hkey = hkey_w(w_hkey, space) buf, buflen = convert_to_regdata(space, w_value, typ) try: - ret = rwinreg.RegSetValueEx(hkey, value_name, 0, typ, buf, buflen) + with rffi.scoped_unicode2wcharp(value_name) as wide_vn: + c_vn = rffi.cast(rffi.CCHARP, wide_vn) + ret = rwinreg.RegSetValueEx(hkey, c_vn, 0, typ, buf, buflen) finally: lltype.free(buf, flavor='raw') if ret != 0: @@ -437,38 +440,40 @@ if space.is_w(w_subkey, space.w_None): subkey = None else: - subkey = space.text_w(w_subkey) + subkey = space.unicode_w(w_subkey) null_dword = lltype.nullptr(rwin32.LPDWORD.TO) - with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retDataSize: - ret = rwinreg.RegQueryValueEx(hkey, subkey, null_dword, null_dword, - None, retDataSize) - bufSize = intmask(retDataSize[0]) - if ret == rwinreg.ERROR_MORE_DATA: - bufSize = 256 - elif ret != 0: - raiseWindowsError(space, ret, 'RegQueryValueEx') + with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: + c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) + with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retDataSize: + ret = rwinreg.RegQueryValueEx(hkey, c_subkey, null_dword, null_dword, + None, retDataSize) + bufSize = intmask(retDataSize[0]) + if ret == rwinreg.ERROR_MORE_DATA: + bufSize = 256 + elif ret != 0: + raiseWindowsError(space, ret, 'RegQueryValueEx') - while True: - with lltype.scoped_alloc(rffi.CCHARP.TO, bufSize) as databuf: - with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retType: + while True: + with lltype.scoped_alloc(rffi.CCHARP.TO, bufSize) as databuf: + with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retType: - ret = rwinreg.RegQueryValueEx(hkey, subkey, null_dword, - retType, databuf, retDataSize) - if ret == rwinreg.ERROR_MORE_DATA: - # Resize and retry - bufSize *= 2 - retDataSize[0] = rffi.cast(rwin32.DWORD, bufSize) - continue - if ret != 0: - raiseWindowsError(space, ret, 'RegQueryValueEx') - length = intmask(retDataSize[0]) - return space.newtuple([ - convert_from_regdata(space, databuf, - length, retType[0]), - space.newint(intmask(retType[0])), + ret = rwinreg.RegQueryValueEx(hkey, c_subkey, null_dword, + retType, databuf, retDataSize) + if ret == rwinreg.ERROR_MORE_DATA: + # Resize and retry + bufSize *= 2 + retDataSize[0] = rffi.cast(rwin32.DWORD, bufSize) + continue + if ret != 0: + raiseWindowsError(space, ret, 'RegQueryValueEx') + length = intmask(retDataSize[0]) + return space.newtuple([ + convert_from_regdata(space, databuf, + length, retType[0]), + space.newint(intmask(retType[0])), ]) - at unwrap_spec(subkey="text") + at unwrap_spec(subkey="unicode") def CreateKey(space, w_hkey, subkey): """key = CreateKey(key, sub_key) - Creates or opens the specified key. @@ -482,14 +487,16 @@ The return value is the handle of the opened key. If the function fails, an exception is raised.""" hkey = hkey_w(w_hkey, space) - with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: - ret = rwinreg.RegCreateKey(hkey, subkey, rethkey) - if ret != 0: - raiseWindowsError(space, ret, 'CreateKey') - return W_HKEY(space, rethkey[0]) + with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: + c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) + with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: + ret = rwinreg.RegCreateKey(hkey, c_subkey, rethkey) + if ret != 0: + raiseWindowsError(space, ret, 'CreateKey') + return W_HKEY(space, rethkey[0]) - at unwrap_spec(sub_key="text", reserved=int, access=rffi.r_uint) -def CreateKeyEx(space, w_key, sub_key, reserved=0, access=rwinreg.KEY_WRITE): + at unwrap_spec(subkey="unicode", reserved=int, access=rffi.r_uint) +def CreateKeyEx(space, w_key, subkey, reserved=0, access=rwinreg.KEY_WRITE): """key = CreateKey(key, sub_key) - Creates or opens the specified key. key is an already open key, or one of the predefined HKEY_* constants @@ -502,17 +509,19 @@ The return value is the handle of the opened key. If the function fails, an exception is raised.""" hkey = hkey_w(w_key, space) - with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: - ret = rwinreg.RegCreateKeyEx(hkey, sub_key, reserved, None, 0, - access, None, rethkey, - lltype.nullptr(rwin32.LPDWORD.TO)) - if ret != 0: - raiseWindowsError(space, ret, 'CreateKeyEx') - return W_HKEY(space, rethkey[0]) + with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: + c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) + with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: + ret = rwinreg.RegCreateKeyEx(hkey, c_subkey, reserved, None, 0, + access, None, rethkey, + lltype.nullptr(rwin32.LPDWORD.TO)) + if ret != 0: + raiseWindowsError(space, ret, 'CreateKeyEx') + return W_HKEY(space, rethkey[0]) - at unwrap_spec(subkey="text") + at unwrap_spec(subkey="unicode") def DeleteKey(space, w_hkey, subkey): - """DeleteKey(key, sub_key) - Deletes the specified key. + """DeleteKey(key, subkey) - Deletes the specified key. key is an already open key, or any one of the predefined HKEY_* constants. sub_key is a string that must be a subkey of the key identified by the key parameter. @@ -523,11 +532,13 @@ If the method succeeds, the entire key, including all of its values, is removed. If the method fails, an EnvironmentError exception is raised.""" hkey = hkey_w(w_hkey, space) - ret = rwinreg.RegDeleteKey(hkey, subkey) - if ret != 0: - raiseWindowsError(space, ret, 'RegDeleteKey') + with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: + c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) + ret = rwinreg.RegDeleteKey(hkey, c_subkey) + if ret != 0: + raiseWindowsError(space, ret, 'RegDeleteKey') - at unwrap_spec(subkey="text") + at unwrap_spec(subkey="unicode") def DeleteValue(space, w_hkey, subkey): """DeleteValue(key, value) - Removes a named value from a registry key. @@ -538,7 +549,7 @@ if ret != 0: raiseWindowsError(space, ret, 'RegDeleteValue') - at unwrap_spec(sub_key="text", reserved=int, access=rffi.r_uint) + at unwrap_spec(sub_key="unicode", reserved=int, access=rffi.r_uint) def OpenKey(space, w_key, sub_key, reserved=0, access=rwinreg.KEY_READ): """key = OpenKey(key, sub_key, res = 0, sam = KEY_READ) - Opens the specified key. @@ -551,11 +562,13 @@ The result is a new handle to the specified key If the function fails, an EnvironmentError exception is raised.""" hkey = hkey_w(w_key, space) - with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: - ret = rwinreg.RegOpenKeyEx(hkey, sub_key, reserved, access, rethkey) - if ret != 0: - raiseWindowsError(space, ret, 'RegOpenKeyEx') - return W_HKEY(space, rethkey[0]) + with rffi.scoped_unicode2wcharp(sub_key) as wide_subkey: + c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) + with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: + ret = rwinreg.RegOpenKeyEx(hkey, c_subkey, reserved, access, rethkey) + if ret != 0: + raiseWindowsError(space, ret, 'RegOpenKeyEx') + return W_HKEY(space, rethkey[0]) @unwrap_spec(index=int) def EnumValue(space, w_hkey, index): @@ -614,7 +627,7 @@ length = intmask(retDataSize[0]) return space.newtuple([ - space.newtext(rffi.charp2str(valuebuf)), + space.newunicode(rffi.wcharp2unicode(valuebuf)), convert_from_regdata(space, databuf, length, retType[0]), space.newint(intmask(retType[0])), @@ -647,7 +660,7 @@ lltype.nullptr(rwin32.PFILETIME.TO)) if ret != 0: raiseWindowsError(space, ret, 'RegEnumKeyEx') - return space.newtext(rffi.charp2str(buf)) + return space.newunicode(rffi.wcharp2unicode(rffi.cast(CWCHARP, buf))) def QueryInfoKey(space, w_hkey): """tuple = QueryInfoKey(key) - Returns information about a key. @@ -726,7 +739,7 @@ raise oefmt(space.w_NotImplementedError, "not implemented on this platform") - at unwrap_spec(sub_key="text", reserved=int, access=rffi.r_uint) + at unwrap_spec(sub_key="unicode", reserved=int, access=rffi.r_uint) def DeleteKeyEx(space, w_key, sub_key, reserved=0, access=rwinreg.KEY_WOW64_64KEY): """DeleteKeyEx(key, sub_key, sam, res) - Deletes the specified key. diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py --- a/pypy/module/_winreg/test/test_winreg.py +++ b/pypy/module/_winreg/test/test_winreg.py @@ -19,7 +19,7 @@ canSaveKey = len(ret) > 0 class AppTestHKey: - #spaceconfig = dict(usemodules=('_winreg',)) + spaceconfig = dict(usemodules=('_winreg',)) def test_repr(self): import winreg @@ -27,12 +27,12 @@ assert str(k) == "" class AppTestFfi: - #spaceconfig = dict(usemodules=('_winreg',)) + spaceconfig = dict(usemodules=('_winreg',)) def setup_class(cls): - import _winreg + import _winreg as winreg space = cls.space - cls.root_key = _winreg.HKEY_CURRENT_USER + cls.root_key = winreg.HKEY_CURRENT_USER cls.test_key_name = "SOFTWARE\\Pypy Registry Test Key - Delete Me" cls.w_root_key = space.wrap(cls.root_key) cls.w_test_key_name = space.wrap(cls.test_key_name) @@ -40,22 +40,22 @@ cls.w_tmpfilename = space.wrap(str(udir.join('winreg-temp'))) test_data = [ - ("Int Value", 0xFEDCBA98, _winreg.REG_DWORD), - ("Str Value", "A string Value", _winreg.REG_SZ), - ("Unicode Value", u"A unicode Value", _winreg.REG_SZ), - ("Str Expand", "The path is %path%", _winreg.REG_EXPAND_SZ), - ("Multi Str", ["Several", "string", u"values"], _winreg.REG_MULTI_SZ), + ("Int Value", 0xFEDCBA98, winreg.REG_DWORD), + ("Str Value", b"A string Value", winreg.REG_SZ), + ("Unicode Value", u"A unicode Value", winreg.REG_SZ), + ("Str Expand", u"The path is %path%", winreg.REG_EXPAND_SZ), + ("Multi Str", [b"Several", u"string", u"values"], winreg.REG_MULTI_SZ), ] cls.w_test_data = w_test_data = space.wrap(test_data) w_btest = space.newtuple([space.wrap("Raw data"), space.newbytes("binary\x00data"), - space.wrap(_winreg.REG_BINARY)]) + space.wrap(winreg.REG_BINARY)]) w_test_data.append(w_btest) def teardown_class(cls): - import winreg + import _winreg try: - winreg.DeleteKey(cls.root_key, cls.test_key_name) + _winreg.DeleteKey(cls.root_key, cls.test_key_name) except WindowsError: pass @@ -67,14 +67,14 @@ def test_simple_write(self): from winreg import SetValue, QueryValue, REG_SZ - value = "Some Default value" + value = u"Some Default value" SetValue(self.root_key, self.test_key_name, REG_SZ, value) assert QueryValue(self.root_key, self.test_key_name) == value def test_CreateKey(self): from winreg import CreateKey, QueryInfoKey key = CreateKey(self.root_key, self.test_key_name) - sub_key = CreateKey(key, "sub_key") + sub_key = CreateKey(key, u"sub_key") nkeys, nvalues, since_mod = QueryInfoKey(key) assert nkeys == 1 @@ -86,7 +86,7 @@ from winreg import CreateKeyEx, QueryInfoKey from winreg import KEY_ALL_ACCESS, KEY_READ key = CreateKeyEx(self.root_key, self.test_key_name, 0, KEY_ALL_ACCESS) - sub_key = CreateKeyEx(key, "sub_key", 0, KEY_READ) + sub_key = CreateKeyEx(key, u"sub_key", 0, KEY_READ) nkeys, nvalues, since_mod = QueryInfoKey(key) assert nkeys == 1 @@ -97,7 +97,7 @@ def test_close(self): from winreg import OpenKey, CloseKey, FlushKey, QueryInfoKey key = OpenKey(self.root_key, self.test_key_name) - sub_key = OpenKey(key, "sub_key") + sub_key = OpenKey(key, u"sub_key") int_sub_key = int(sub_key) FlushKey(sub_key) @@ -119,7 +119,7 @@ def test_with(self): from winreg import OpenKey with OpenKey(self.root_key, self.test_key_name) as key: - with OpenKey(key, "sub_key") as sub_key: + with OpenKey(key, u"sub_key") as sub_key: assert key.handle != 0 assert sub_key.handle != 0 assert key.handle == 0 @@ -142,7 +142,7 @@ def test_SetValueEx(self): from winreg import CreateKey, SetValueEx, REG_BINARY, REG_DWORD key = CreateKey(self.root_key, self.test_key_name) - sub_key = CreateKey(key, "sub_key") + sub_key = CreateKey(key, u"sub_key") SetValueEx(sub_key, 'Int Value', 0, REG_DWORD, None) SetValueEx(sub_key, 'Int Value', 0, REG_DWORD, 45) for name, value, type in self.test_data: diff --git a/rpython/rlib/rwinreg.py b/rpython/rlib/rwinreg.py --- a/rpython/rlib/rwinreg.py +++ b/rpython/rlib/rwinreg.py @@ -47,7 +47,7 @@ HKEY = rwin32.HANDLE PHKEY = rffi.CArrayPtr(HKEY) REGSAM = rwin32.DWORD -suffix = 'A' +suffix = 'W' RegSetValue = external( 'RegSetValue' + suffix, From pypy.commits at gmail.com Tue Jan 2 03:00:34 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 02 Jan 2018 00:00:34 -0800 (PST) Subject: [pypy-commit] pypy py3-winreg: fix translation Message-ID: <5a4b3c22.559a1c0a.6e0a2.1337@mx.google.com> Author: Matti Picus Branch: py3-winreg Changeset: r93609:683cb7587096 Date: 2018-01-02 09:55 +0200 http://bitbucket.org/pypy/pypy/changeset/683cb7587096/ Log: fix translation diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py --- a/pypy/module/_winreg/interp_winreg.py +++ b/pypy/module/_winreg/interp_winreg.py @@ -110,8 +110,8 @@ elif isinstance(w_hkey, W_HKEY): return w_hkey.hkey elif space.isinstance_w(w_hkey, space.w_int): - return rffi.cast(rwinreg.HKEY, space.int_w(w_hkey)) - elif space.isinstance_w(w_hkey, space.w_long): + if space.is_true(space.lt(w_hkey, space.newint(0))): + return rffi.cast(rwinreg.HKEY, space.int_w(w_hkey)) return rffi.cast(rwinreg.HKEY, space.uint_w(w_hkey)) else: raise oefmt(space.w_TypeError, "The object is not a PyHKEY object") @@ -170,9 +170,13 @@ The docs imply key must be in the HKEY_USER or HKEY_LOCAL_MACHINE tree""" # XXX should filename use space.fsencode_w? hkey = hkey_w(w_hkey, space) - ret = rwinreg.RegLoadKey(hkey, subkey, filename) - if ret != 0: - raiseWindowsError(space, ret, 'RegLoadKey') + with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: + c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) + with rffi.scoped_unicode2wcharp(filename) as wide_filename: + c_filename = rffi.cast(rffi.CCHARP, wide_filename) + ret = rwinreg.RegLoadKey(hkey, c_subkey, c_filename) + if ret != 0: + raiseWindowsError(space, ret, 'RegLoadKey') @unwrap_spec(filename="unicode") def SaveKey(space, w_hkey, filename): @@ -189,9 +193,11 @@ The caller of this method must possess the SeBackupPrivilege security privilege. This function passes NULL for security_attributes to the API.""" hkey = hkey_w(w_hkey, space) - ret = rwinreg.RegSaveKey(hkey, filename, None) - if ret != 0: - raiseWindowsError(space, ret, 'RegSaveKey') + with rffi.scoped_unicode2wcharp(filename) as wide_filename: + c_filename = rffi.cast(rffi.CCHARP, wide_filename) + ret = rwinreg.RegSaveKey(hkey, c_filename, None) + if ret != 0: + raiseWindowsError(space, ret, 'RegSaveKey') @unwrap_spec(typ=int, value="unicode") def SetValue(space, w_hkey, w_subkey, typ, value): @@ -267,6 +273,9 @@ return space.newunicode(rffi.wcharp2unicoden(wide_buf, length)) def convert_to_regdata(space, w_value, typ): + ''' + returns CCHARP, int + ''' buf = None if typ == rwinreg.REG_DWORD: @@ -320,7 +329,7 @@ with rffi.scoped_unicode2wcharp(string) as wchr: c_str = rffi.cast(rffi.CCHARP, wchr) for i in range(len(string)): - buf[buflen + i] = wchr[i] + buf[buflen + i] = c_str[i] buflen += len(string) + 1 buf[buflen - 1] = '\0' buflen += 1 @@ -358,14 +367,14 @@ elif typ == rwinreg.REG_SZ or typ == rwinreg.REG_EXPAND_SZ: if not buflen: - s = "" + s = u"" else: # may or may not have a trailing NULL in the buffer. buf = rffi.cast(rffi.CWCHARP, buf) if buf[buflen - 1] == '\x00': buflen -= 1 s = rffi.wcharp2unicoden(buf, buflen) - w_s = space.newbytes(s) + w_s = space.newunicode(s) return w_s elif typ == rwinreg.REG_MULTI_SZ: @@ -380,7 +389,7 @@ i += 1 if len(s) == 0: break - s = ''.join(s) + s = u''.join(s) l.append(space.newunicode(s)) i += 1 return space.newlist(l) @@ -545,9 +554,11 @@ key is an already open key, or any one of the predefined HKEY_* constants. value is a string that identifies the value to remove.""" hkey = hkey_w(w_hkey, space) - ret = rwinreg.RegDeleteValue(hkey, subkey) - if ret != 0: - raiseWindowsError(space, ret, 'RegDeleteValue') + with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: + c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) + ret = rwinreg.RegDeleteValue(hkey, c_subkey) + if ret != 0: + raiseWindowsError(space, ret, 'RegDeleteValue') @unwrap_spec(sub_key="unicode", reserved=int, access=rffi.r_uint) def OpenKey(space, w_key, sub_key, reserved=0, access=rwinreg.KEY_READ): @@ -603,15 +614,16 @@ bufDataSize = intmask(retDataSize[0]) bufValueSize = intmask(retValueSize[0]) - with lltype.scoped_alloc(rffi.CCHARP.TO, + with lltype.scoped_alloc(rffi.CWCHARP.TO, intmask(retValueSize[0])) as valuebuf: while True: with lltype.scoped_alloc(rffi.CCHARP.TO, bufDataSize) as databuf: with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retType: + c_valuebuf = rffi.cast(rffi.CCHARP, valuebuf) ret = rwinreg.RegEnumValue( - hkey, index, valuebuf, retValueSize, + hkey, index, c_valuebuf, retValueSize, null_dword, retType, databuf, retDataSize) if ret == rwinreg.ERROR_MORE_DATA: # Resize and retry @@ -660,7 +672,7 @@ lltype.nullptr(rwin32.PFILETIME.TO)) if ret != 0: raiseWindowsError(space, ret, 'RegEnumKeyEx') - return space.newunicode(rffi.wcharp2unicode(rffi.cast(CWCHARP, buf))) + return space.newunicode(rffi.wcharp2unicode(rffi.cast(rffi.CWCHARP, buf))) def QueryInfoKey(space, w_hkey): """tuple = QueryInfoKey(key) - Returns information about a key. From pypy.commits at gmail.com Tue Jan 2 05:26:40 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 02 Jan 2018 02:26:40 -0800 (PST) Subject: [pypy-commit] pypy py3-winreg: fixes, multibytes is two bytes, memoryview can be stored now Message-ID: <5a4b5e60.51a0df0a.bb55f.42f0@mx.google.com> Author: Matti Picus Branch: py3-winreg Changeset: r93610:a9b00a31a070 Date: 2018-01-02 12:25 +0200 http://bitbucket.org/pypy/pypy/changeset/a9b00a31a070/ Log: fixes, multibytes is two bytes, memoryview can be stored now diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py --- a/pypy/module/_winreg/interp_winreg.py +++ b/pypy/module/_winreg/interp_winreg.py @@ -297,7 +297,7 @@ else: buf = rffi.unicode2wcharp(space.unicode_w(w_value)) buf = rffi.cast(rffi.CCHARP, buf) - buflen = space.len_w(w_value) + 1 + buflen = (space.len_w(w_value) * 2) + 1 elif typ == rwinreg.REG_MULTI_SZ: if space.is_w(w_value, space.w_None): @@ -315,12 +315,12 @@ w_item = space.next(w_iter) item = space.unicode_w(w_item) strings.append(item) - buflen += len(item) + 1 + buflen += 2 * (len(item) + 1) except OperationError as e: if not e.match(space, space.w_StopIteration): raise # re-raise other app-level exceptions break - buflen += 1 + buflen += 2 buf = lltype.malloc(rffi.CCHARP.TO, buflen, flavor='raw') # Now copy data @@ -328,12 +328,14 @@ for string in strings: with rffi.scoped_unicode2wcharp(string) as wchr: c_str = rffi.cast(rffi.CCHARP, wchr) - for i in range(len(string)): + for i in range(len(string) * 2): buf[buflen + i] = c_str[i] - buflen += len(string) + 1 + buflen += (len(string) + 1) * 2 buf[buflen - 1] = '\0' - buflen += 1 + buf[buflen - 2] = '\0' + buflen += 2 buf[buflen - 1] = '\0' + buf[buflen - 2] = '\0' else: # REG_BINARY and ALL unknown data types. if space.is_w(w_value, space.w_None): @@ -371,6 +373,7 @@ else: # may or may not have a trailing NULL in the buffer. buf = rffi.cast(rffi.CWCHARP, buf) + buflen /= 2 if buf[buflen - 1] == '\x00': buflen -= 1 s = rffi.wcharp2unicoden(buf, buflen) @@ -380,11 +383,13 @@ elif typ == rwinreg.REG_MULTI_SZ: if not buflen: return space.newlist([]) + buf = rffi.cast(rffi.CWCHARP, buf) + buflen /= 2 i = 0 l = [] while i < buflen and buf[i]: s = [] - while i < buflen and buf[i] != '\0': + while i < buflen and buf[i] != u'\0': s.append(buf[i]) i += 1 if len(s) == 0: @@ -504,8 +509,8 @@ raiseWindowsError(space, ret, 'CreateKey') return W_HKEY(space, rethkey[0]) - at unwrap_spec(subkey="unicode", reserved=int, access=rffi.r_uint) -def CreateKeyEx(space, w_key, subkey, reserved=0, access=rwinreg.KEY_WRITE): + at unwrap_spec(sub_key="unicode", reserved=int, access=rffi.r_uint) +def CreateKeyEx(space, w_key, sub_key, reserved=0, access=rwinreg.KEY_WRITE): """key = CreateKey(key, sub_key) - Creates or opens the specified key. key is an already open key, or one of the predefined HKEY_* constants @@ -518,8 +523,8 @@ The return value is the handle of the opened key. If the function fails, an exception is raised.""" hkey = hkey_w(w_key, space) - with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: - c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) + with rffi.scoped_unicode2wcharp(sub_key) as wide_sub_key: + c_subkey = rffi.cast(rffi.CCHARP, wide_sub_key) with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: ret = rwinreg.RegCreateKeyEx(hkey, c_subkey, reserved, None, 0, access, None, rethkey, diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py --- a/pypy/module/_winreg/test/test_winreg.py +++ b/pypy/module/_winreg/test/test_winreg.py @@ -42,8 +42,8 @@ test_data = [ ("Int Value", 0xFEDCBA98, winreg.REG_DWORD), ("Str Value", b"A string Value", winreg.REG_SZ), - ("Unicode Value", u"A unicode Value", winreg.REG_SZ), - ("Str Expand", u"The path is %path%", winreg.REG_EXPAND_SZ), + ("Unicode Value", "A unicode Value", winreg.REG_SZ), + ("Str Expand", "The path is %path%", winreg.REG_EXPAND_SZ), ("Multi Str", [b"Several", u"string", u"values"], winreg.REG_MULTI_SZ), ] cls.w_test_data = w_test_data = space.wrap(test_data) @@ -140,6 +140,7 @@ assert 0, "Did not raise" def test_SetValueEx(self): + # this test leaves open keys. If it fails, others will too from winreg import CreateKey, SetValueEx, REG_BINARY, REG_DWORD key = CreateKey(self.root_key, self.test_key_name) sub_key = CreateKey(key, u"sub_key") @@ -147,10 +148,9 @@ SetValueEx(sub_key, 'Int Value', 0, REG_DWORD, 45) for name, value, type in self.test_data: SetValueEx(sub_key, name, 0, type, value) - exc = raises(TypeError, SetValueEx, sub_key, 'test_name', None, - REG_BINARY, memoryview('abc')) - assert str(exc.value) == ("Objects of type 'memoryview' can not " - "be used as binary registry values") + # cannot wrap a memoryview in setup_class for test_data + SetValueEx(sub_key, u'test_name', None, + REG_BINARY, memoryview(b'abc')) def test_readValues(self): from winreg import OpenKey, EnumValue, QueryValueEx, EnumKey @@ -163,27 +163,31 @@ data = EnumValue(sub_key, index) except EnvironmentError as e: break - assert data in self.test_data + if data[0] != 'test_name': + # cannot wrap a memoryview in setup_class for test_data + assert data in self.test_data index = index + 1 - assert index == len(self.test_data) + assert index == len(self.test_data) + 1 for name, value, type in self.test_data: result = QueryValueEx(sub_key, name) assert result == (value, type) if type == REG_SZ or type == REG_EXPAND_SZ: - assert isinstance(result[0], unicode) # not string + assert not isinstance(result[0], bytes) assert EnumKey(key, 0) == "sub_key" raises(EnvironmentError, EnumKey, key, 1) def test_delete(self): # must be run after test_SetValueEx - from _winreg import OpenKey, KEY_ALL_ACCESS, DeleteValue, DeleteKey + from winreg import OpenKey, KEY_ALL_ACCESS, DeleteValue, DeleteKey key = OpenKey(self.root_key, self.test_key_name, 0, KEY_ALL_ACCESS) sub_key = OpenKey(key, "sub_key", 0, KEY_ALL_ACCESS) for name, value, type in self.test_data: DeleteValue(sub_key, name) + # cannot wrap a memoryview in setup_class for test_data + DeleteValue(sub_key, 'test_name') DeleteKey(key, "sub_key") From pypy.commits at gmail.com Tue Jan 2 13:32:01 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 02 Jan 2018 10:32:01 -0800 (PST) Subject: [pypy-commit] pypy py3-winreg: CPython does not accept None and we don't either Message-ID: <5a4bd021.e6361c0a.e898e.db3e@mx.google.com> Author: Matti Picus Branch: py3-winreg Changeset: r93611:c1d65fb9e61d Date: 2018-01-02 20:31 +0200 http://bitbucket.org/pypy/pypy/changeset/c1d65fb9e61d/ Log: CPython does not accept None and we don't either diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py --- a/pypy/module/_winreg/test/test_winreg.py +++ b/pypy/module/_winreg/test/test_winreg.py @@ -246,7 +246,7 @@ skip("access denied to registry key " "(are you running in a non-interactive session?)") raise - QueryValueEx(HKEY_PERFORMANCE_DATA, None) + QueryValueEx(HKEY_PERFORMANCE_DATA, 'Global') def test_reflection_unsupported(self): import sys From pypy.commits at gmail.com Tue Jan 2 13:48:35 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 02 Jan 2018 10:48:35 -0800 (PST) Subject: [pypy-commit] pypy py3-winreg: fix WindowsError instantiation Message-ID: <5a4bd403.19a0df0a.7a9e.dd70@mx.google.com> Author: Matti Picus Branch: py3-winreg Changeset: r93612:b2b7567f4963 Date: 2018-01-02 20:47 +0200 http://bitbucket.org/pypy/pypy/changeset/b2b7567f4963/ Log: fix WindowsError instantiation diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py --- a/pypy/module/_winreg/interp_winreg.py +++ b/pypy/module/_winreg/interp_winreg.py @@ -9,9 +9,10 @@ def raiseWindowsError(space, errcode, context): message = rwin32.FormatError(errcode) + w_errcode = space.newint(errcode) raise OperationError(space.w_WindowsError, - space.newtuple([space.newint(errcode), - space.newtext(message)])) + space.newtuple([w_errcode, space.newtext(message), + space.w_None, w_errcode])) class W_HKEY(W_Root): def __init__(self, space, hkey): From pypy.commits at gmail.com Tue Jan 2 15:45:58 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 02 Jan 2018 12:45:58 -0800 (PST) Subject: [pypy-commit] buildbot default: whoops, isRPython needs log Message-ID: <5a4bef86.8e9cdf0a.63e79.1f4b@mx.google.com> Author: Matti Picus Branch: Changeset: r1054:469f7461ec34 Date: 2018-01-02 22:44 +0200 http://bitbucket.org/pypy/buildbot/changeset/469f7461ec34/ Log: whoops, isRPython needs log diff --git a/bot2/pypybuildbot/util.py b/bot2/pypybuildbot/util.py --- a/bot2/pypybuildbot/util.py +++ b/bot2/pypybuildbot/util.py @@ -1,5 +1,6 @@ import os import socket +from buildbot.buildslave.base import log def we_are_debugging(): return socket.gethostname() != 'baroquesoftware' From pypy.commits at gmail.com Tue Jan 2 16:53:05 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 02 Jan 2018 13:53:05 -0800 (PST) Subject: [pypy-commit] pypy default: reflow, remove outdated instructions no longer needed Message-ID: <5a4bff41.cbbadf0a.3c695.8f4b@mx.google.com> Author: Matti Picus Branch: Changeset: r93613:c19f313b2e7f Date: 2018-01-02 23:50 +0200 http://bitbucket.org/pypy/pypy/changeset/c19f313b2e7f/ Log: reflow, remove outdated instructions no longer needed diff --git a/rpython/doc/arm.rst b/rpython/doc/arm.rst --- a/rpython/doc/arm.rst +++ b/rpython/doc/arm.rst @@ -8,7 +8,9 @@ compilation toolchain. To translate an RPython program for ARM we can either -translate directly on an ARM device following the normal translation steps. Unfortunately this is not really feasible on most ARM machines. The alternative is to cross-translate using a cross-compilation toolchain. +translate directly on an ARM device following the normal translation steps. +Unfortunately this is not really feasible on most ARM machines. The +alternative is to cross-translate using a cross-compilation toolchain. To cross-translate we run the translation on a more powerful (usually x86) machine and generate a binary for ARM using a cross-compiler to compile @@ -24,7 +26,8 @@ The tools required to cross translate from a Linux based host to an ARM based Linux target are: - A checkout of PyPy (default branch). -- The GCC ARM cross compiler (on Ubuntu it is the ``gcc-arm-linux-gnueabi package``) but other toolchains should also work. +- The GCC ARM cross compiler (on Ubuntu it is the ``gcc-arm-linux-gnueabi package``) but other + toolchains should also work. - Scratchbox 2, a cross-compilation engine (``scratchbox2`` Ubuntu package). - A 32-bit PyPy or Python. - And the following (or corresponding) packages need to be installed to create an ARM based chroot: @@ -150,15 +153,14 @@ pypy ~/path_to_pypy_checkout/rpython/bin/rpython -O2 --platform=arm target.py -If everything worked correctly this should yield an ARM binary. Running this binary in the ARM chroot or on an ARM device should produce the output ``"Hello World"``. +If everything worked correctly this should yield an ARM binary. Running this +binary in the ARM chroot or on an ARM device should produce the output +``"Hello World"``. To translate the full python pypy interpreter with a jit, you can cd into pypy/goal and call :: - pypy /rpython/bin/rpython -Ojit --platform=arm --gcrootfinder=shadowstack --jit-backend=arm targetpypystandalone.py + pypy /rpython/bin/rpython -Ojit --platform=arm targetpypystandalone.py -The gcrootfinder option is needed to work around `issue 1377`_ and the jit-backend works around `issue 1376`_ -.. _issue 1377: https://bitbucket.org/pypy/pypy/issue/1377 -.. _issue 1376: https://bitbucket.org/pypy/pypy/issue/1376 From pypy.commits at gmail.com Wed Jan 3 13:05:20 2018 From: pypy.commits at gmail.com (arigo) Date: Wed, 03 Jan 2018 10:05:20 -0800 (PST) Subject: [pypy-commit] pypy default: Invent a dummy thread id, if we ask for one during translation Message-ID: <5a4d1b60.08691c0a.8e86.b0db@mx.google.com> Author: Armin Rigo Branch: Changeset: r93614:05955e010535 Date: 2018-01-03 19:04 +0100 http://bitbucket.org/pypy/pypy/changeset/05955e010535/ Log: Invent a dummy thread id, if we ask for one during translation and we're running a host with no threads diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -106,14 +106,20 @@ if we_are_translated(): return tlfield_thread_ident.getraw() else: - import thread + try: + import thread + except ImportError: + return 42 return thread.get_ident() def get_or_make_ident(): if we_are_translated(): return tlfield_thread_ident.get_or_make_raw() else: - import thread + try: + import thread + except ImportError: + return 42 return thread.get_ident() @specialize.arg(0) From pypy.commits at gmail.com Wed Jan 3 17:32:53 2018 From: pypy.commits at gmail.com (amauryfa) Date: Wed, 03 Jan 2018 14:32:53 -0800 (PST) Subject: [pypy-commit] pypy default: First stab at parser.tuple2st() Message-ID: <5a4d5a15.512f1c0a.24dcb.ace2@mx.google.com> Author: Amaury Forgeot d'Arc Branch: Changeset: r93615:28aa6e61df25 Date: 2017-12-28 22:49 +0100 http://bitbucket.org/pypy/pypy/changeset/28aa6e61df25/ Log: First stab at parser.tuple2st() diff --git a/pypy/module/parser/__init__.py b/pypy/module/parser/__init__.py --- a/pypy/module/parser/__init__.py +++ b/pypy/module/parser/__init__.py @@ -24,5 +24,7 @@ 'ASTType' : 'pyparser.W_STType', 'compilest' : 'pyparser.compilest', 'compileast' : 'pyparser.compilest', - 'ParserError' : 'space.new_exception_class("parser.ParserError")', + 'tuple2st' : 'pyparser.tuple2st', + 'sequence2st' : 'pyparser.tuple2st', + 'ParserError' : 'pyparser.get_error(space)', } diff --git a/pypy/module/parser/pyparser.py b/pypy/module/parser/pyparser.py --- a/pypy/module/parser/pyparser.py +++ b/pypy/module/parser/pyparser.py @@ -8,6 +8,14 @@ from rpython.rlib.objectmodel import specialize +class Cache: + def __init__(self, space): + self.error = space.new_exception_class("parser.ParserError") + +def get_error(space): + return space.fromcache(Cache).error + + class W_STType(W_Root): def __init__(self, tree, mode): self.tree = tree @@ -114,3 +122,59 @@ @unwrap_spec(w_st=W_STType) def compilest(space, w_st, __args__): return space.call_args(space.getattr(w_st, space.newtext("compile")), __args__) + + +def raise_parser_error(space, w_tuple, message): + raise OperationError(get_error(space), space.newtuple( + [w_tuple, space.newtext("Illegal component tuple.")])) + + +def get_node_type(space, w_tuple): + try: + w_type = space.getitem(w_tuple, space.newint(0)) + return space.int_w(w_type) + except OperationError: + raise_parser_error(space, w_tuple, "Illegal component tuple.") + +class NodeState: + def __init__(self): + self.lineno = 0 + +def build_node_tree(space, w_tuple): + type = get_node_type(space, w_tuple) + node_state = NodeState() + if 0 <= type < 256: + # The tuple is simple, but it doesn't start with a start symbol. + # Raise an exception now and be done with it. + raise_parser_error(space, w_tuple, + "Illegal syntax-tree; cannot start with terminal symbol.") + node = pyparse.parser.Nonterminal(type, []) + build_node_children(space, w_tuple, node, node_state) + return node + +def build_node_children(space, w_tuple, node, node_state): + for w_elem in space.unpackiterable(w_tuple)[1:]: + type = get_node_type(space, w_elem) + if type < 256: # Terminal node + length = space.len_w(w_elem) + if length == 2: + _, w_obj = space.unpackiterable(w_elem, 2) + elif length == 3: + _, w_obj, w_lineno = space.unpackiterable(w_elem, 3) + else: + raise_error(space, "terminal nodes must have 2 or 3 entries") + strn = space.text_w(w_obj) + child = pyparse.parser.Terminal(type, strn, node_state.lineno, 0) + else: + child = pyparse.parser.Nonterminal(type, []) + node.append_child(child) + if type >= 256: # Nonterminal node + build_node_children(space, w_elem, child, node_state) + elif type == pyparse.pygram.tokens.NEWLINE: + node_state.lineno += 1 + + +def tuple2st(space, w_sequence): + # Convert the tree to the internal form before checking it + tree = build_node_tree(space, w_sequence) + return W_STType(tree, 'eval') diff --git a/pypy/module/parser/test/test_parser.py b/pypy/module/parser/test/test_parser.py --- a/pypy/module/parser/test/test_parser.py +++ b/pypy/module/parser/test/test_parser.py @@ -56,3 +56,18 @@ def test_error(self): assert repr(self.m.ParserError) == "" + + def test_roundtrip(self): + def roundtrip(f, s): + st1 = f(s) + t = st1.totuple() + st2 = self.m.sequence2st(t) + assert t == st2.totuple() + + def check_expr(s): + roundtrip(self.m.expr, s) + def check_suite(s): + roundtrip(self.m.suite, s) + + check_expr("foo(1)") + check_suite("def f(): yield 1") From pypy.commits at gmail.com Wed Jan 3 17:32:55 2018 From: pypy.commits at gmail.com (amauryfa) Date: Wed, 03 Jan 2018 14:32:55 -0800 (PST) Subject: [pypy-commit] pypy default: parser.sequence2st: add validation of the passed tuple. Message-ID: <5a4d5a17.cbbadf0a.1998b.2bb3@mx.google.com> Author: Amaury Forgeot d'Arc Branch: Changeset: r93616:749063799a58 Date: 2018-01-02 00:53 +0100 http://bitbucket.org/pypy/pypy/changeset/749063799a58/ Log: parser.sequence2st: add validation of the passed tuple. Do it the 2016 way, by walking the grammar DFA, instead of a ton of custom validation code. diff --git a/pypy/module/parser/pyparser.py b/pypy/module/parser/pyparser.py --- a/pypy/module/parser/pyparser.py +++ b/pypy/module/parser/pyparser.py @@ -124,9 +124,13 @@ return space.call_args(space.getattr(w_st, space.newtext("compile")), __args__) -def raise_parser_error(space, w_tuple, message): +def parser_error(space, w_tuple, message): raise OperationError(get_error(space), space.newtuple( - [w_tuple, space.newtext("Illegal component tuple.")])) + [w_tuple, space.newtext(message)])) + +def parse_error(space, message): + return OperationError(get_error(space), + space.newtext(message)) def get_node_type(space, w_tuple): @@ -134,7 +138,7 @@ w_type = space.getitem(w_tuple, space.newint(0)) return space.int_w(w_type) except OperationError: - raise_parser_error(space, w_tuple, "Illegal component tuple.") + raise parser_error(space, w_tuple, "Illegal component tuple.") class NodeState: def __init__(self): @@ -146,7 +150,7 @@ if 0 <= type < 256: # The tuple is simple, but it doesn't start with a start symbol. # Raise an exception now and be done with it. - raise_parser_error(space, w_tuple, + raise parser_error(space, w_tuple, "Illegal syntax-tree; cannot start with terminal symbol.") node = pyparse.parser.Nonterminal(type, []) build_node_children(space, w_tuple, node, node_state) @@ -162,7 +166,8 @@ elif length == 3: _, w_obj, w_lineno = space.unpackiterable(w_elem, 3) else: - raise_error(space, "terminal nodes must have 2 or 3 entries") + raise parse_error( + space, "terminal nodes must have 2 or 3 entries") strn = space.text_w(w_obj) child = pyparse.parser.Terminal(type, strn, node_state.lineno, 0) else: @@ -174,7 +179,36 @@ node_state.lineno += 1 +def validate_node(space, tree): + assert tree.type >= 256 + type = tree.type - 256 + parser = pyparse.PythonParser(space) + if type >= len(parser.grammar.dfas): + raise parse_error(space, "Unrecognized node type %d." % type) + dfa = parser.grammar.dfas[type] + # Run the DFA for this nonterminal + states, first = dfa + arcs, is_accepting = states[0] + for pos in range(tree.num_children()): + ch = tree.get_child(pos) + for i, next_state in arcs: + label = parser.grammar.labels[i] + if label == ch.type: + # The child is acceptable; validate it recursively + if ch.type >= 256: + validate_node(space, ch) + # Update the state, and move on to the next child. + arcs, is_accepting = states[next_state] + break + else: + raise parse_error(space, "Illegal node") + if not is_accepting: + raise parse_error(space, "Illegal number of children for %d node" % + tree.type) + + def tuple2st(space, w_sequence): # Convert the tree to the internal form before checking it tree = build_node_tree(space, w_sequence) + validate_node(space, tree) return W_STType(tree, 'eval') diff --git a/pypy/module/parser/test/test_parser.py b/pypy/module/parser/test/test_parser.py --- a/pypy/module/parser/test/test_parser.py +++ b/pypy/module/parser/test/test_parser.py @@ -71,3 +71,19 @@ check_expr("foo(1)") check_suite("def f(): yield 1") + + def test_bad_tree(self): + import parser + # from import a + tree = \ + (257, + (267, + (268, + (269, + (281, + (283, (1, 'from'), (1, 'import'), + (286, (284, (1, 'fred')))))), + (4, ''))), + (4, ''), (0, '')) + raises(parser.ParserError, + parser.sequence2st, tree) From pypy.commits at gmail.com Wed Jan 3 17:54:57 2018 From: pypy.commits at gmail.com (amauryfa) Date: Wed, 03 Jan 2018 14:54:57 -0800 (PST) Subject: [pypy-commit] pypy py3.6: Allow undescores in int() literals. Message-ID: <5a4d5f41.f8b8df0a.7a542.c8db@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r93622:ff9805f6a687 Date: 2018-01-03 22:51 +0100 http://bitbucket.org/pypy/pypy/changeset/ff9805f6a687/ Log: Allow undescores in int() literals. diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py --- a/pypy/objspace/std/intobject.py +++ b/pypy/objspace/std/intobject.py @@ -810,7 +810,7 @@ def _string_to_int_or_long(space, w_inttype, w_source, string, base=10): try: - value = string_to_int(string, base) + value = string_to_int(string, base, allow_underscores=True) except ParseStringError as e: raise wrap_parsestringerror(space, e, w_source) except ParseStringOverflowError as e: From pypy.commits at gmail.com Wed Jan 3 17:54:53 2018 From: pypy.commits at gmail.com (amauryfa) Date: Wed, 03 Jan 2018 14:54:53 -0800 (PST) Subject: [pypy-commit] pypy py3.6: Attempt to parse numbers with underscores Message-ID: <5a4d5f3d.85c9df0a.b7054.dd34@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r93620:c123301c02cc Date: 2018-01-03 12:17 +0100 http://bitbucket.org/pypy/pypy/changeset/c123301c02cc/ Log: Attempt to parse numbers with underscores diff --git a/pypy/interpreter/pyparser/dfa_generated.py b/pypy/interpreter/pyparser/dfa_generated.py --- a/pypy/interpreter/pyparser/dfa_generated.py +++ b/pypy/interpreter/pyparser/dfa_generated.py @@ -7,10 +7,14 @@ accepts = [True, True, True, True, True, True, True, True, True, True, True, False, True, True, True, True, True, False, False, False, True, False, False, - False, True, False, True, False, True, False, False, True, False, False, True, False, False, - True, True, True, False, False, True, False, - False, False, True] + True, False, False, True, False, False, True, + False, False, True, False, True, False, True, + False, True, False, False, False, False, True, + True, False, False, False, False, True, False, + True, False, True, False, True, False, True, True, + False, True, False, True, False, False, True, + True, True, True, True] states = [ # 0 {'\t': 0, '\n': 15, '\x0c': 0, @@ -110,21 +114,21 @@ 'v': 1, 'w': 1, 'x': 1, 'y': 1, 'z': 1, '\x80': 1}, # 5 - {'.': 26, '0': 24, '1': 25, '2': 25, - '3': 25, '4': 25, '5': 25, '6': 25, - '7': 25, '8': 25, '9': 25, 'B': 23, - 'E': 27, 'J': 15, 'O': 22, 'X': 21, - 'b': 23, 'e': 27, 'j': 15, 'o': 22, - 'x': 21}, + {'.': 27, '0': 24, '1': 26, '2': 26, + '3': 26, '4': 26, '5': 26, '6': 26, + '7': 26, '8': 26, '9': 26, 'B': 23, + 'E': 28, 'J': 15, 'O': 22, 'X': 21, + '_': 25, 'b': 23, 'e': 28, 'j': 15, + 'o': 22, 'x': 21}, # 6 - {'.': 26, '0': 6, '1': 6, '2': 6, + {'.': 27, '0': 6, '1': 6, '2': 6, '3': 6, '4': 6, '5': 6, '6': 6, - '7': 6, '8': 6, '9': 6, 'E': 27, - 'J': 15, 'e': 27, 'j': 15}, + '7': 6, '8': 6, '9': 6, 'E': 28, + 'J': 15, '_': 29, 'e': 28, 'j': 15}, # 7 - {'.': 29, '0': 28, '1': 28, '2': 28, - '3': 28, '4': 28, '5': 28, '6': 28, - '7': 28, '8': 28, '9': 28}, + {'.': 31, '0': 30, '1': 30, '2': 30, + '3': 30, '4': 30, '5': 30, '6': 30, + '7': 30, '8': 30, '9': 30}, # 8 {'*': 14, '=': 15}, # 9 @@ -144,107 +148,240 @@ # 16 {'\n': 15}, # 17 - {automata.DEFAULT: 33, '\n': 30, - '\r': 30, "'": 31, '\\': 32}, + {automata.DEFAULT: 35, '\n': 32, + '\r': 32, "'": 33, '\\': 34}, # 18 - {automata.DEFAULT: 36, '\n': 30, - '\r': 30, '"': 34, '\\': 35}, + {automata.DEFAULT: 38, '\n': 32, + '\r': 32, '"': 36, '\\': 37}, # 19 {'\n': 15, '\r': 16}, # 20 - {automata.DEFAULT: 20, '\n': 30, '\r': 30}, + {automata.DEFAULT: 20, '\n': 32, '\r': 32}, # 21 - {'0': 37, '1': 37, '2': 37, '3': 37, - '4': 37, '5': 37, '6': 37, '7': 37, - '8': 37, '9': 37, 'A': 37, 'B': 37, - 'C': 37, 'D': 37, 'E': 37, 'F': 37, - 'a': 37, 'b': 37, 'c': 37, 'd': 37, - 'e': 37, 'f': 37}, + {'0': 39, '1': 39, '2': 39, '3': 39, + '4': 39, '5': 39, '6': 39, '7': 39, + '8': 39, '9': 39, 'A': 39, 'B': 39, + 'C': 39, 'D': 39, 'E': 39, 'F': 39, + '_': 40, 'a': 39, 'b': 39, 'c': 39, + 'd': 39, 'e': 39, 'f': 39}, # 22 - {'0': 38, '1': 38, '2': 38, '3': 38, - '4': 38, '5': 38, '6': 38, '7': 38}, + {'0': 41, '1': 41, '2': 41, '3': 41, + '4': 41, '5': 41, '6': 41, '7': 41, + '_': 42}, # 23 - {'0': 39, '1': 39}, + {'0': 43, '1': 43, '_': 44}, # 24 - {'.': 26, '0': 24, '1': 25, '2': 25, - '3': 25, '4': 25, '5': 25, '6': 25, - '7': 25, '8': 25, '9': 25, 'E': 27, - 'J': 15, 'e': 27, 'j': 15}, + {'.': 27, '0': 24, '1': 26, '2': 26, + '3': 26, '4': 26, '5': 26, '6': 26, + '7': 26, '8': 26, '9': 26, 'E': 28, + 'J': 15, '_': 25, 'e': 28, 'j': 15}, # 25 - {'.': 26, '0': 25, '1': 25, '2': 25, - '3': 25, '4': 25, '5': 25, '6': 25, - '7': 25, '8': 25, '9': 25, 'E': 27, - 'J': 15, 'e': 27, 'j': 15}, + {'0': 45, '1': 46, '2': 46, '3': 46, + '4': 46, '5': 46, '6': 46, '7': 46, + '8': 46, '9': 46}, # 26 - {'0': 26, '1': 26, '2': 26, '3': 26, - '4': 26, '5': 26, '6': 26, '7': 26, - '8': 26, '9': 26, 'E': 40, 'J': 15, - 'e': 40, 'j': 15}, + {'.': 27, '0': 26, '1': 26, '2': 26, + '3': 26, '4': 26, '5': 26, '6': 26, + '7': 26, '8': 26, '9': 26, 'E': 28, + 'J': 15, '_': 47, 'e': 28, 'j': 15}, # 27 - {'+': 41, '-': 41, '0': 42, '1': 42, - '2': 42, '3': 42, '4': 42, '5': 42, - '6': 42, '7': 42, '8': 42, '9': 42}, + {'0': 27, '1': 27, '2': 27, '3': 27, + '4': 27, '5': 27, '6': 27, '7': 27, + '8': 27, '9': 27, 'E': 48, 'J': 15, + 'e': 48, 'j': 15}, # 28 - {'0': 28, '1': 28, '2': 28, '3': 28, - '4': 28, '5': 28, '6': 28, '7': 28, - '8': 28, '9': 28, 'E': 40, 'J': 15, - 'e': 40, 'j': 15}, + {'+': 49, '-': 49, '0': 50, '1': 50, + '2': 50, '3': 50, '4': 50, '5': 50, + '6': 50, '7': 50, '8': 50, '9': 50}, # 29 + {'0': 51, '1': 51, '2': 51, '3': 51, + '4': 51, '5': 51, '6': 51, '7': 51, + '8': 51, '9': 51}, + # 30 + {'0': 30, '1': 30, '2': 30, '3': 30, + '4': 30, '5': 30, '6': 30, '7': 30, + '8': 30, '9': 30, 'E': 48, 'J': 15, + '_': 52, 'e': 48, 'j': 15}, + # 31 {'.': 15}, - # 30 + # 32 {}, - # 31 + # 33 {"'": 15}, - # 32 - {automata.DEFAULT: 43, '\n': 15, '\r': 16}, - # 33 - {automata.DEFAULT: 33, '\n': 30, - '\r': 30, "'": 15, '\\': 32}, # 34 + {automata.DEFAULT: 53, '\n': 15, '\r': 16}, + # 35 + {automata.DEFAULT: 35, '\n': 32, + '\r': 32, "'": 15, '\\': 34}, + # 36 {'"': 15}, - # 35 - {automata.DEFAULT: 44, '\n': 15, '\r': 16}, - # 36 - {automata.DEFAULT: 36, '\n': 30, - '\r': 30, '"': 15, '\\': 35}, # 37 - {'0': 37, '1': 37, '2': 37, '3': 37, - '4': 37, '5': 37, '6': 37, '7': 37, - '8': 37, '9': 37, 'A': 37, 'B': 37, - 'C': 37, 'D': 37, 'E': 37, 'F': 37, - 'a': 37, 'b': 37, 'c': 37, 'd': 37, - 'e': 37, 'f': 37}, + {automata.DEFAULT: 54, '\n': 15, '\r': 16}, # 38 - {'0': 38, '1': 38, '2': 38, '3': 38, - '4': 38, '5': 38, '6': 38, '7': 38}, + {automata.DEFAULT: 38, '\n': 32, + '\r': 32, '"': 15, '\\': 37}, # 39 - {'0': 39, '1': 39}, + {'0': 39, '1': 39, '2': 39, '3': 39, + '4': 39, '5': 39, '6': 39, '7': 39, + '8': 39, '9': 39, 'A': 39, 'B': 39, + 'C': 39, 'D': 39, 'E': 39, 'F': 39, + '_': 55, 'a': 39, 'b': 39, 'c': 39, + 'd': 39, 'e': 39, 'f': 39}, # 40 - {'+': 45, '-': 45, '0': 46, '1': 46, - '2': 46, '3': 46, '4': 46, '5': 46, - '6': 46, '7': 46, '8': 46, '9': 46}, + {'0': 56, '1': 56, '2': 56, '3': 56, + '4': 56, '5': 56, '6': 56, '7': 56, + '8': 56, '9': 56, 'A': 56, 'B': 56, + 'C': 56, 'D': 56, 'E': 56, 'F': 56, + 'a': 56, 'b': 56, 'c': 56, 'd': 56, + 'e': 56, 'f': 56}, # 41 - {'0': 42, '1': 42, '2': 42, '3': 42, - '4': 42, '5': 42, '6': 42, '7': 42, - '8': 42, '9': 42}, + {'0': 41, '1': 41, '2': 41, '3': 41, + '4': 41, '5': 41, '6': 41, '7': 41, + '_': 57}, # 42 - {'0': 42, '1': 42, '2': 42, '3': 42, - '4': 42, '5': 42, '6': 42, '7': 42, - '8': 42, '9': 42, 'J': 15, 'j': 15}, + {'0': 58, '1': 58, '2': 58, '3': 58, + '4': 58, '5': 58, '6': 58, '7': 58}, # 43 - {automata.DEFAULT: 43, '\n': 30, - '\r': 30, "'": 15, '\\': 32}, + {'0': 43, '1': 43, '_': 59}, # 44 - {automata.DEFAULT: 44, '\n': 30, - '\r': 30, '"': 15, '\\': 35}, + {'0': 60, '1': 60}, # 45 + {'.': 27, '0': 45, '1': 46, '2': 46, + '3': 46, '4': 46, '5': 46, '6': 46, + '7': 46, '8': 46, '9': 46, 'E': 28, + 'J': 15, '_': 25, 'e': 28, 'j': 15}, + # 46 + {'.': 27, '0': 46, '1': 46, '2': 46, + '3': 46, '4': 46, '5': 46, '6': 46, + '7': 46, '8': 46, '9': 46, 'E': 28, + 'J': 15, '_': 47, 'e': 28, 'j': 15}, + # 47 {'0': 46, '1': 46, '2': 46, '3': 46, '4': 46, '5': 46, '6': 46, '7': 46, '8': 46, '9': 46}, - # 46 - {'0': 46, '1': 46, '2': 46, '3': 46, - '4': 46, '5': 46, '6': 46, '7': 46, - '8': 46, '9': 46, 'J': 15, 'j': 15}, + # 48 + {'+': 61, '-': 61, '0': 62, '1': 62, + '2': 62, '3': 62, '4': 62, '5': 62, + '6': 62, '7': 62, '8': 62, '9': 62}, + # 49 + {'0': 50, '1': 50, '2': 50, '3': 50, + '4': 50, '5': 50, '6': 50, '7': 50, + '8': 50, '9': 50}, + # 50 + {'0': 50, '1': 50, '2': 50, '3': 50, + '4': 50, '5': 50, '6': 50, '7': 50, + '8': 50, '9': 50, 'J': 15, '_': 63, + 'j': 15}, + # 51 + {'.': 27, '0': 51, '1': 51, '2': 51, + '3': 51, '4': 51, '5': 51, '6': 51, + '7': 51, '8': 51, '9': 51, 'E': 28, + 'J': 15, '_': 29, 'e': 28, 'j': 15}, + # 52 + {'0': 64, '1': 64, '2': 64, '3': 64, + '4': 64, '5': 64, '6': 64, '7': 64, + '8': 64, '9': 64}, + # 53 + {automata.DEFAULT: 53, '\n': 32, + '\r': 32, "'": 15, '\\': 34}, + # 54 + {automata.DEFAULT: 54, '\n': 32, + '\r': 32, '"': 15, '\\': 37}, + # 55 + {'0': 65, '1': 65, '2': 65, '3': 65, + '4': 65, '5': 65, '6': 65, '7': 65, + '8': 65, '9': 65, 'A': 65, 'B': 65, + 'C': 65, 'D': 65, 'E': 65, 'F': 65, + 'a': 65, 'b': 65, 'c': 65, 'd': 65, + 'e': 65, 'f': 65}, + # 56 + {'0': 56, '1': 56, '2': 56, '3': 56, + '4': 56, '5': 56, '6': 56, '7': 56, + '8': 56, '9': 56, 'A': 56, 'B': 56, + 'C': 56, 'D': 56, 'E': 56, 'F': 56, + '_': 66, 'a': 56, 'b': 56, 'c': 56, + 'd': 56, 'e': 56, 'f': 56}, + # 57 + {'0': 67, '1': 67, '2': 67, '3': 67, + '4': 67, '5': 67, '6': 67, '7': 67}, + # 58 + {'0': 58, '1': 58, '2': 58, '3': 58, + '4': 58, '5': 58, '6': 58, '7': 58, + '_': 68}, + # 59 + {'0': 69, '1': 69}, + # 60 + {'0': 60, '1': 60, '_': 70}, + # 61 + {'0': 62, '1': 62, '2': 62, '3': 62, + '4': 62, '5': 62, '6': 62, '7': 62, + '8': 62, '9': 62}, + # 62 + {'0': 62, '1': 62, '2': 62, '3': 62, + '4': 62, '5': 62, '6': 62, '7': 62, + '8': 62, '9': 62, 'J': 15, '_': 71, + 'j': 15}, + # 63 + {'0': 72, '1': 72, '2': 72, '3': 72, + '4': 72, '5': 72, '6': 72, '7': 72, + '8': 72, '9': 72}, + # 64 + {'0': 64, '1': 64, '2': 64, '3': 64, + '4': 64, '5': 64, '6': 64, '7': 64, + '8': 64, '9': 64, 'E': 48, 'J': 15, + '_': 52, 'e': 48, 'j': 15}, + # 65 + {'0': 65, '1': 65, '2': 65, '3': 65, + '4': 65, '5': 65, '6': 65, '7': 65, + '8': 65, '9': 65, 'A': 65, 'B': 65, + 'C': 65, 'D': 65, 'E': 65, 'F': 65, + '_': 55, 'a': 65, 'b': 65, 'c': 65, + 'd': 65, 'e': 65, 'f': 65}, + # 66 + {'0': 73, '1': 73, '2': 73, '3': 73, + '4': 73, '5': 73, '6': 73, '7': 73, + '8': 73, '9': 73, 'A': 73, 'B': 73, + 'C': 73, 'D': 73, 'E': 73, 'F': 73, + 'a': 73, 'b': 73, 'c': 73, 'd': 73, + 'e': 73, 'f': 73}, + # 67 + {'0': 67, '1': 67, '2': 67, '3': 67, + '4': 67, '5': 67, '6': 67, '7': 67, + '_': 57}, + # 68 + {'0': 74, '1': 74, '2': 74, '3': 74, + '4': 74, '5': 74, '6': 74, '7': 74}, + # 69 + {'0': 69, '1': 69, '_': 59}, + # 70 + {'0': 75, '1': 75}, + # 71 + {'0': 76, '1': 76, '2': 76, '3': 76, + '4': 76, '5': 76, '6': 76, '7': 76, + '8': 76, '9': 76}, + # 72 + {'0': 72, '1': 72, '2': 72, '3': 72, + '4': 72, '5': 72, '6': 72, '7': 72, + '8': 72, '9': 72, 'J': 15, '_': 63, + 'j': 15}, + # 73 + {'0': 73, '1': 73, '2': 73, '3': 73, + '4': 73, '5': 73, '6': 73, '7': 73, + '8': 73, '9': 73, 'A': 73, 'B': 73, + 'C': 73, 'D': 73, 'E': 73, 'F': 73, + '_': 66, 'a': 73, 'b': 73, 'c': 73, + 'd': 73, 'e': 73, 'f': 73}, + # 74 + {'0': 74, '1': 74, '2': 74, '3': 74, + '4': 74, '5': 74, '6': 74, '7': 74, + '_': 68}, + # 75 + {'0': 75, '1': 75, '_': 70}, + # 76 + {'0': 76, '1': 76, '2': 76, '3': 76, + '4': 76, '5': 76, '6': 76, '7': 76, + '8': 76, '9': 76, 'J': 15, '_': 71, + 'j': 15}, ] pseudoDFA = automata.DFA(states, accepts) diff --git a/pypy/interpreter/pyparser/gendfa.py b/pypy/interpreter/pyparser/gendfa.py --- a/pypy/interpreter/pyparser/gendfa.py +++ b/pypy/interpreter/pyparser/gendfa.py @@ -60,28 +60,43 @@ # Digits def makeDigits (): return groupStr(states, "0123456789") + def makeDigitsChain (digits="0123456789", first=None, + allow_leading_underscore=False): + if first is None: + first = digits + if allow_leading_underscore: + return group(states, + makeDigitsChain(digits=digits), + chain(states, + newArcPair(states, "_"), + makeDigitsChain(digits=digits))) + return chain(states, + groupStr(states, first), + any(states, groupStr(states, digits)), + any(states, + chain(states, + newArcPair(states, "_"), + atleastonce(states, groupStr(states, digits))))) + # ____________________________________________________________ # Integer numbers hexNumber = chain(states, newArcPair(states, "0"), groupStr(states, "xX"), - atleastonce(states, - groupStr(states, "0123456789abcdefABCDEF"))) + makeDigitsChain("0123456789abcdefABCDEF", + allow_leading_underscore=True)) octNumber = chain(states, newArcPair(states, "0"), groupStr(states, "oO"), - groupStr(states, "01234567"), - any(states, groupStr(states, "01234567"))) + makeDigitsChain("01234567", + allow_leading_underscore=True)) binNumber = chain(states, newArcPair(states, "0"), groupStr(states, "bB"), - atleastonce(states, groupStr(states, "01"))) - decNumber = chain(states, - groupStr(states, "123456789"), - any(states, makeDigits())) - zero = chain(states, - newArcPair(states, "0"), - any(states, newArcPair(states, "0"))) + makeDigitsChain("01", + allow_leading_underscore=True)) + decNumber = makeDigitsChain(first="123456789") + zero = makeDigitsChain("0") intNumber = group(states, hexNumber, octNumber, binNumber, decNumber, zero) # ____________________________________________________________ # Exponents @@ -89,29 +104,34 @@ return chain(states, groupStr(states, "eE"), maybe(states, groupStr(states, "+-")), - atleastonce(states, makeDigits())) + makeDigitsChain()) + # ____________________________________________________________ # Floating point numbers + def makePointFloat (): + return group(states, + chain(states, + makeDigitsChain(), + newArcPair(states, "."), + any(states, makeDigits())), + chain(states, + newArcPair(states, "."), + makeDigitsChain())) def makeFloat (): - pointFloat = chain(states, - group(states, - chain(states, - atleastonce(states, makeDigits()), - newArcPair(states, "."), - any(states, makeDigits())), - chain(states, - newArcPair(states, "."), - atleastonce(states, makeDigits()))), - maybe(states, makeExp())) + pointFloat = group(states, + makePointFloat(), + chain(states, + makePointFloat(), + makeExp())) expFloat = chain(states, - atleastonce(states, makeDigits()), + makeDigitsChain(), makeExp()) return group(states, pointFloat, expFloat) # ____________________________________________________________ # Imaginary numbers imagNumber = group(states, chain(states, - atleastonce(states, makeDigits()), + makeDigitsChain(), groupStr(states, "jJ")), chain(states, makeFloat(), diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py --- a/pypy/interpreter/pyparser/test/test_pyparse.py +++ b/pypy/interpreter/pyparser/test/test_pyparse.py @@ -191,8 +191,90 @@ async with a: pass""") py.test.raises(SyntaxError, self.parse, 'def foo(): async with a: pass') - - + + def test_number_underscores(self): + VALID_UNDERSCORE_LITERALS = [ + '0_0_0', + '4_2', + '1_0000_0000', + '0b1001_0100', + '0xffff_ffff', + '0o5_7_7', + '1_00_00.5', + '1_00_00.5e5', + '1_00_00e5_1', + '1e1_0', + '.1_4', + '.1_4e1', + '0b_0', + '0x_f', + '0o_5', + '1_00_00j', + '1_00_00.5j', + '1_00_00e5_1j', + '.1_4j', + '(1_2.5+3_3j)', + '(.5_6j)', + ] + INVALID_UNDERSCORE_LITERALS = [ + # Trailing underscores: + '0_', + '42_', + '1.4j_', + '0x_', + '0b1_', + '0xf_', + '0o5_', + '0 if 1_Else 1', + # Underscores in the base selector: + '0_b0', + '0_xf', + '0_o5', + # Old-style octal, still disallowed: + '0_7', + '09_99', + # Multiple consecutive underscores: + '4_______2', + '0.1__4', + '0.1__4j', + '0b1001__0100', + '0xffff__ffff', + '0x___', + '0o5__77', + '1e1__0', + '1e1__0j', + # Underscore right before a dot: + '1_.4', + '1_.4j', + # Underscore right after a dot: + '1._4', + '1._4j', + '._5', + '._5j', + # Underscore right after a sign: + '1.0e+_1', + '1.0e+_1j', + # Underscore right before j: + '1.4_j', + '1.4e5_j', + # Underscore right before e: + '1_e1', + '1.4_e1', + '1.4_e1j', + # Underscore right after e: + '1e_1', + '1.4e_1', + '1.4e_1j', + # Complex cases with parens: + '(1+1.5_j_)', + '(1+1.5_j)', + ] + for x in VALID_UNDERSCORE_LITERALS: + tree = self.parse(x) + for x in INVALID_UNDERSCORE_LITERALS: + print x + raises(SyntaxError, self.parse, "x = %s" % x) + class TestPythonParserWithSpace: From pypy.commits at gmail.com Wed Jan 3 17:54:49 2018 From: pypy.commits at gmail.com (amauryfa) Date: Wed, 03 Jan 2018 14:54:49 -0800 (PST) Subject: [pypy-commit] pypy py3.6: Add sha3 (aka. Keccak) hashes to hashlib. Message-ID: <5a4d5f39.8681df0a.e58ff.dce0@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r93618:88331f108204 Date: 2017-12-25 18:11 +0100 http://bitbucket.org/pypy/pypy/changeset/88331f108204/ Log: Add sha3 (aka. Keccak) hashes to hashlib. diff too long, truncating to 2000 out of 5634 lines diff --git a/lib-python/3/hashlib.py b/lib-python/3/hashlib.py --- a/lib-python/3/hashlib.py +++ b/lib-python/3/hashlib.py @@ -57,10 +57,8 @@ # always available algorithm is added. __always_supported = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'blake2b', 'blake2s', - # FIXME the following algorithms were added in - # cpython3.6 but are missing in pypy - # 'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', - # 'shake_128', 'shake_256' + 'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', + 'shake_128', 'shake_256' ) diff --git a/lib_pypy/_sha3/__init__.py b/lib_pypy/_sha3/__init__.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_sha3/__init__.py @@ -0,0 +1,107 @@ +from ._sha3_cffi import ffi as _ffi, lib as _lib +import codecs + +SHA3_MAX_DIGESTSIZE = 64 # 64 Bytes (512 Bits) for 224 to 512 +SHA3_LANESIZE = (20 * 8) # ExtractLane needs max uint64_t[20] extra. + +class _sha3: + _keccak_init = None # Overridden in subclasses + + def __new__(cls, string=None): + self = super().__new__(cls) + self._hash_state = _ffi.new("Keccak_HashInstance*") + + cls._keccak_init(self._hash_state) + + if string: + self.update(string) + return self + + def update(self, string): + buf = _ffi.from_buffer(string) + res = _lib.Keccak_HashUpdate(self._hash_state, buf, len(buf) * 8) + + def digest(self): + digest = _ffi.new("char[]", + SHA3_MAX_DIGESTSIZE + SHA3_LANESIZE) + state_copy = _ffi.new("Keccak_HashInstance*") + _ffi.memmove(state_copy, self._hash_state, + _ffi.sizeof("Keccak_HashInstance")) + if _lib.Keccak_HashFinal(state_copy, digest) != _lib.SUCCESS: + raise RuntimeError("internal error in SHA3 Final()") + return _ffi.unpack(digest, self._hash_state.fixedOutputLength // 8) + + def hexdigest(self): + return codecs.encode(self.digest(), 'hex').decode() + + def copy(self): + copy = super().__new__(type(self)) + copy._hash_state = _ffi.new("Keccak_HashInstance*") + _ffi.memmove(copy._hash_state, self._hash_state, + _ffi.sizeof("Keccak_HashInstance")) + return copy + + @property + def digest_size(self): + return self._hash_state.fixedOutputLength // 8 + + @property + def block_size(self): + return self._hash_state.sponge.rate // 8 + + @property + def _capacity_bits(self): + return 1600 - self._hash_state.sponge.rate + + @property + def _rate_bits(self): + return self._hash_state.sponge.rate + + @property + def _suffix(self): + return bytes([self._hash_state.delimitedSuffix]) + + +class _shake(_sha3): + def digest(self, length): + # ExtractLane needs at least SHA3_MAX_DIGESTSIZE + SHA3_LANESIZE and + # SHA_LANESIZE extra space. + digest = _ffi.new("char[]", length + SHA3_LANESIZE) + # Get the raw (binary) digest value + state_copy = _ffi.new("Keccak_HashInstance*") + _ffi.memmove(state_copy, self._hash_state, + _ffi.sizeof("Keccak_HashInstance")) + if _lib.Keccak_HashFinal(state_copy, digest) != _lib.SUCCESS: + raise RuntimeError("internal error in SHA3 Final()") + if _lib.Keccak_HashSqueeze(state_copy, digest, length * 8) != _lib.SUCCESS: + raise RuntimeError("internal error in SHA3 Squeeze()") + return _ffi.unpack(digest, length) + + def hexdigest(self, length): + return codecs.encode(self.digest(length), 'hex').decode() + + +class sha3_224(_sha3): + name = "sha3_224" + _keccak_init = _lib.Keccak_HashInitialize_SHA3_224 + +class sha3_256(_sha3): + name = "sha3_256" + _keccak_init = _lib.Keccak_HashInitialize_SHA3_256 + +class sha3_384(_sha3): + name = "sha3_384" + _keccak_init = _lib.Keccak_HashInitialize_SHA3_384 + +class sha3_512(_sha3): + name = "sha3_512" + _keccak_init = _lib.Keccak_HashInitialize_SHA3_512 + +class shake_128(_shake): + name = "shake_128" + _keccak_init = _lib.Keccak_HashInitialize_SHAKE128 + +class shake_256(_shake): + name = "shake_256" + _keccak_init = _lib.Keccak_HashInitialize_SHAKE256 + diff --git a/lib_pypy/_sha3/_sha3_build.py b/lib_pypy/_sha3/_sha3_build.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_sha3/_sha3_build.py @@ -0,0 +1,78 @@ +import os +import sys + +from cffi import FFI + + +ffi = FFI() +ffi.cdef(""" +typedef struct { + unsigned int rate; + ...; +} KeccakWidth1600_SpongeInstance; + +typedef struct { + KeccakWidth1600_SpongeInstance sponge; + unsigned int fixedOutputLength; + unsigned char delimitedSuffix; + ...; +} Keccak_HashInstance; + +typedef enum { SUCCESS = ..., FAIL = ..., BAD_HASHLEN = ... } HashReturn; +typedef int... DataLength; +typedef unsigned char BitSequence; + +HashReturn Keccak_HashInitialize_SHA3_224(Keccak_HashInstance*); +HashReturn Keccak_HashInitialize_SHA3_256(Keccak_HashInstance*); +HashReturn Keccak_HashInitialize_SHA3_384(Keccak_HashInstance*); +HashReturn Keccak_HashInitialize_SHA3_512(Keccak_HashInstance*); +HashReturn Keccak_HashInitialize_SHAKE128(Keccak_HashInstance*); +HashReturn Keccak_HashInitialize_SHAKE256(Keccak_HashInstance*); + +HashReturn Keccak_HashUpdate(Keccak_HashInstance *, const BitSequence *, DataLength); +HashReturn Keccak_HashFinal(Keccak_HashInstance *, BitSequence *); +HashReturn Keccak_HashSqueeze(Keccak_HashInstance *hashInstance, BitSequence *data, DataLength databitlen); +""") + +_libdir = os.path.join(os.path.dirname(__file__), 'kcp') + +if sys.byteorder == 'big': + # opt64 is not yet supported on big endian platforms + keccakOpt = 32 +elif sys.maxsize > 2**32: + keccakOpt = 64 +else: + keccakOpt = 32 + +ffi.set_source( + '_sha3_cffi', +("#define KeccakOpt %d\n" % keccakOpt) + +""" +/* we are only interested in KeccakP1600 */ +#define KeccakP200_excluded 1 +#define KeccakP400_excluded 1 +#define KeccakP800_excluded 1 + +#if KeccakOpt == 64 + /* 64bit platforms with unsigned int64 */ + typedef uint64_t UINT64; + typedef unsigned char UINT8; +#endif + +/* inline all Keccak dependencies */ +#include "kcp/KeccakHash.h" +#include "kcp/KeccakSponge.h" +#include "kcp/KeccakHash.c" +#include "kcp/KeccakSponge.c" +#if KeccakOpt == 64 + #include "kcp/KeccakP-1600-opt64.c" +#elif KeccakOpt == 32 + #include "kcp/KeccakP-1600-inplace32BI.c" +#endif +""", + include_dirs=[_libdir], +) + +if __name__ == '__main__': + os.chdir(os.path.dirname(__file__)) + ffi.compile() diff --git a/lib_pypy/_sha3/kcp/KeccakHash.c b/lib_pypy/_sha3/kcp/KeccakHash.c new file mode 100644 --- /dev/null +++ b/lib_pypy/_sha3/kcp/KeccakHash.c @@ -0,0 +1,82 @@ +/* +Implementation by the Keccak, Keyak and Ketje Teams, namely, Guido Bertoni, +Joan Daemen, Michaël Peeters, Gilles Van Assche and Ronny Van Keer, hereby +denoted as "the implementer". + +For more information, feedback or questions, please refer to our websites: +http://keccak.noekeon.org/ +http://keyak.noekeon.org/ +http://ketje.noekeon.org/ + +To the extent possible under law, the implementer has waived all copyright +and related or neighboring rights to the source code in this file. +http://creativecommons.org/publicdomain/zero/1.0/ +*/ + +#include +#include "KeccakHash.h" + +/* ---------------------------------------------------------------- */ + +HashReturn Keccak_HashInitialize(Keccak_HashInstance *instance, unsigned int rate, unsigned int capacity, unsigned int hashbitlen, unsigned char delimitedSuffix) +{ + HashReturn result; + + if (delimitedSuffix == 0) + return FAIL; + result = (HashReturn)KeccakWidth1600_SpongeInitialize(&instance->sponge, rate, capacity); + if (result != SUCCESS) + return result; + instance->fixedOutputLength = hashbitlen; + instance->delimitedSuffix = delimitedSuffix; + return SUCCESS; +} + +/* ---------------------------------------------------------------- */ + +HashReturn Keccak_HashUpdate(Keccak_HashInstance *instance, const BitSequence *data, DataLength databitlen) +{ + if ((databitlen % 8) == 0) + return (HashReturn)KeccakWidth1600_SpongeAbsorb(&instance->sponge, data, databitlen/8); + else { + HashReturn ret = (HashReturn)KeccakWidth1600_SpongeAbsorb(&instance->sponge, data, databitlen/8); + if (ret == SUCCESS) { + /* The last partial byte is assumed to be aligned on the least significant bits */ + + unsigned char lastByte = data[databitlen/8]; + /* Concatenate the last few bits provided here with those of the suffix */ + + unsigned short delimitedLastBytes = (unsigned short)((unsigned short)lastByte | ((unsigned short)instance->delimitedSuffix << (databitlen % 8))); + if ((delimitedLastBytes & 0xFF00) == 0x0000) { + instance->delimitedSuffix = delimitedLastBytes & 0xFF; + } + else { + unsigned char oneByte[1]; + oneByte[0] = delimitedLastBytes & 0xFF; + ret = (HashReturn)KeccakWidth1600_SpongeAbsorb(&instance->sponge, oneByte, 1); + instance->delimitedSuffix = (delimitedLastBytes >> 8) & 0xFF; + } + } + return ret; + } +} + +/* ---------------------------------------------------------------- */ + +HashReturn Keccak_HashFinal(Keccak_HashInstance *instance, BitSequence *hashval) +{ + HashReturn ret = (HashReturn)KeccakWidth1600_SpongeAbsorbLastFewBits(&instance->sponge, instance->delimitedSuffix); + if (ret == SUCCESS) + return (HashReturn)KeccakWidth1600_SpongeSqueeze(&instance->sponge, hashval, instance->fixedOutputLength/8); + else + return ret; +} + +/* ---------------------------------------------------------------- */ + +HashReturn Keccak_HashSqueeze(Keccak_HashInstance *instance, BitSequence *data, DataLength databitlen) +{ + if ((databitlen % 8) != 0) + return FAIL; + return (HashReturn)KeccakWidth1600_SpongeSqueeze(&instance->sponge, data, databitlen/8); +} diff --git a/lib_pypy/_sha3/kcp/KeccakHash.h b/lib_pypy/_sha3/kcp/KeccakHash.h new file mode 100644 --- /dev/null +++ b/lib_pypy/_sha3/kcp/KeccakHash.h @@ -0,0 +1,114 @@ +/* +Implementation by the Keccak, Keyak and Ketje Teams, namely, Guido Bertoni, +Joan Daemen, Michaël Peeters, Gilles Van Assche and Ronny Van Keer, hereby +denoted as "the implementer". + +For more information, feedback or questions, please refer to our websites: +http://keccak.noekeon.org/ +http://keyak.noekeon.org/ +http://ketje.noekeon.org/ + +To the extent possible under law, the implementer has waived all copyright +and related or neighboring rights to the source code in this file. +http://creativecommons.org/publicdomain/zero/1.0/ +*/ + +#ifndef _KeccakHashInterface_h_ +#define _KeccakHashInterface_h_ + +#ifndef KeccakP1600_excluded + +#include "KeccakSponge.h" +#include + +typedef unsigned char BitSequence; +typedef size_t DataLength; +typedef enum { SUCCESS = 0, FAIL = 1, BAD_HASHLEN = 2 } HashReturn; + +typedef struct { + KeccakWidth1600_SpongeInstance sponge; + unsigned int fixedOutputLength; + unsigned char delimitedSuffix; +} Keccak_HashInstance; + +/** + * Function to initialize the Keccak[r, c] sponge function instance used in sequential hashing mode. + * @param hashInstance Pointer to the hash instance to be initialized. + * @param rate The value of the rate r. + * @param capacity The value of the capacity c. + * @param hashbitlen The desired number of output bits, + * or 0 for an arbitrarily-long output. + * @param delimitedSuffix Bits that will be automatically appended to the end + * of the input message, as in domain separation. + * This is a byte containing from 0 to 7 bits + * formatted like the @a delimitedData parameter of + * the Keccak_SpongeAbsorbLastFewBits() function. + * @pre One must have r+c=1600 and the rate a multiple of 8 bits in this implementation. + * @return SUCCESS if successful, FAIL otherwise. + */ +HashReturn Keccak_HashInitialize(Keccak_HashInstance *hashInstance, unsigned int rate, unsigned int capacity, unsigned int hashbitlen, unsigned char delimitedSuffix); + +/** Macro to initialize a SHAKE128 instance as specified in the FIPS 202 standard. + */ +#define Keccak_HashInitialize_SHAKE128(hashInstance) Keccak_HashInitialize(hashInstance, 1344, 256, 0, 0x1F) + +/** Macro to initialize a SHAKE256 instance as specified in the FIPS 202 standard. + */ +#define Keccak_HashInitialize_SHAKE256(hashInstance) Keccak_HashInitialize(hashInstance, 1088, 512, 0, 0x1F) + +/** Macro to initialize a SHA3-224 instance as specified in the FIPS 202 standard. + */ +#define Keccak_HashInitialize_SHA3_224(hashInstance) Keccak_HashInitialize(hashInstance, 1152, 448, 224, 0x06) + +/** Macro to initialize a SHA3-256 instance as specified in the FIPS 202 standard. + */ +#define Keccak_HashInitialize_SHA3_256(hashInstance) Keccak_HashInitialize(hashInstance, 1088, 512, 256, 0x06) + +/** Macro to initialize a SHA3-384 instance as specified in the FIPS 202 standard. + */ +#define Keccak_HashInitialize_SHA3_384(hashInstance) Keccak_HashInitialize(hashInstance, 832, 768, 384, 0x06) + +/** Macro to initialize a SHA3-512 instance as specified in the FIPS 202 standard. + */ +#define Keccak_HashInitialize_SHA3_512(hashInstance) Keccak_HashInitialize(hashInstance, 576, 1024, 512, 0x06) + +/** + * Function to give input data to be absorbed. + * @param hashInstance Pointer to the hash instance initialized by Keccak_HashInitialize(). + * @param data Pointer to the input data. + * When @a databitLen is not a multiple of 8, the last bits of data must be + * in the least significant bits of the last byte (little-endian convention). + * @param databitLen The number of input bits provided in the input data. + * @pre In the previous call to Keccak_HashUpdate(), databitlen was a multiple of 8. + * @return SUCCESS if successful, FAIL otherwise. + */ +HashReturn Keccak_HashUpdate(Keccak_HashInstance *hashInstance, const BitSequence *data, DataLength databitlen); + +/** + * Function to call after all input blocks have been input and to get + * output bits if the length was specified when calling Keccak_HashInitialize(). + * @param hashInstance Pointer to the hash instance initialized by Keccak_HashInitialize(). + * If @a hashbitlen was not 0 in the call to Keccak_HashInitialize(), the number of + * output bits is equal to @a hashbitlen. + * If @a hashbitlen was 0 in the call to Keccak_HashInitialize(), the output bits + * must be extracted using the Keccak_HashSqueeze() function. + * @param state Pointer to the state of the sponge function initialized by Init(). + * @param hashval Pointer to the buffer where to store the output data. + * @return SUCCESS if successful, FAIL otherwise. + */ +HashReturn Keccak_HashFinal(Keccak_HashInstance *hashInstance, BitSequence *hashval); + + /** + * Function to squeeze output data. + * @param hashInstance Pointer to the hash instance initialized by Keccak_HashInitialize(). + * @param data Pointer to the buffer where to store the output data. + * @param databitlen The number of output bits desired (must be a multiple of 8). + * @pre Keccak_HashFinal() must have been already called. + * @pre @a databitlen is a multiple of 8. + * @return SUCCESS if successful, FAIL otherwise. + */ +HashReturn Keccak_HashSqueeze(Keccak_HashInstance *hashInstance, BitSequence *data, DataLength databitlen); + +#endif + +#endif diff --git a/lib_pypy/_sha3/kcp/KeccakP-1600-64.macros b/lib_pypy/_sha3/kcp/KeccakP-1600-64.macros new file mode 100644 --- /dev/null +++ b/lib_pypy/_sha3/kcp/KeccakP-1600-64.macros @@ -0,0 +1,2208 @@ +/* +Implementation by the Keccak, Keyak and Ketje Teams, namely, Guido Bertoni, +Joan Daemen, Michaël Peeters, Gilles Van Assche and Ronny Van Keer, hereby +denoted as "the implementer". + +For more information, feedback or questions, please refer to our websites: +http://keccak.noekeon.org/ +http://keyak.noekeon.org/ +http://ketje.noekeon.org/ + +To the extent possible under law, the implementer has waived all copyright +and related or neighboring rights to the source code in this file. +http://creativecommons.org/publicdomain/zero/1.0/ +*/ + +#define declareABCDE \ + UINT64 Aba, Abe, Abi, Abo, Abu; \ + UINT64 Aga, Age, Agi, Ago, Agu; \ + UINT64 Aka, Ake, Aki, Ako, Aku; \ + UINT64 Ama, Ame, Ami, Amo, Amu; \ + UINT64 Asa, Ase, Asi, Aso, Asu; \ + UINT64 Bba, Bbe, Bbi, Bbo, Bbu; \ + UINT64 Bga, Bge, Bgi, Bgo, Bgu; \ + UINT64 Bka, Bke, Bki, Bko, Bku; \ + UINT64 Bma, Bme, Bmi, Bmo, Bmu; \ + UINT64 Bsa, Bse, Bsi, Bso, Bsu; \ + UINT64 Ca, Ce, Ci, Co, Cu; \ + UINT64 Da, De, Di, Do, Du; \ + UINT64 Eba, Ebe, Ebi, Ebo, Ebu; \ + UINT64 Ega, Ege, Egi, Ego, Egu; \ + UINT64 Eka, Eke, Eki, Eko, Eku; \ + UINT64 Ema, Eme, Emi, Emo, Emu; \ + UINT64 Esa, Ese, Esi, Eso, Esu; \ + +#define prepareTheta \ + Ca = Aba^Aga^Aka^Ama^Asa; \ + Ce = Abe^Age^Ake^Ame^Ase; \ + Ci = Abi^Agi^Aki^Ami^Asi; \ + Co = Abo^Ago^Ako^Amo^Aso; \ + Cu = Abu^Agu^Aku^Amu^Asu; \ + +#ifdef UseBebigokimisa +/* --- Code for round, with prepare-theta (lane complementing pattern 'bebigokimisa') */ + +/* --- 64-bit lanes mapped to 64-bit words */ + +#define thetaRhoPiChiIotaPrepareTheta(i, A, E) \ + Da = Cu^ROL64(Ce, 1); \ + De = Ca^ROL64(Ci, 1); \ + Di = Ce^ROL64(Co, 1); \ + Do = Ci^ROL64(Cu, 1); \ + Du = Co^ROL64(Ca, 1); \ +\ + A##ba ^= Da; \ + Bba = A##ba; \ + A##ge ^= De; \ + Bbe = ROL64(A##ge, 44); \ + A##ki ^= Di; \ + Bbi = ROL64(A##ki, 43); \ + A##mo ^= Do; \ + Bbo = ROL64(A##mo, 21); \ + A##su ^= Du; \ + Bbu = ROL64(A##su, 14); \ + E##ba = Bba ^( Bbe | Bbi ); \ + E##ba ^= KeccakF1600RoundConstants[i]; \ + Ca = E##ba; \ + E##be = Bbe ^((~Bbi)| Bbo ); \ + Ce = E##be; \ + E##bi = Bbi ^( Bbo & Bbu ); \ + Ci = E##bi; \ + E##bo = Bbo ^( Bbu | Bba ); \ + Co = E##bo; \ + E##bu = Bbu ^( Bba & Bbe ); \ + Cu = E##bu; \ +\ + A##bo ^= Do; \ + Bga = ROL64(A##bo, 28); \ + A##gu ^= Du; \ + Bge = ROL64(A##gu, 20); \ + A##ka ^= Da; \ + Bgi = ROL64(A##ka, 3); \ + A##me ^= De; \ + Bgo = ROL64(A##me, 45); \ + A##si ^= Di; \ + Bgu = ROL64(A##si, 61); \ + E##ga = Bga ^( Bge | Bgi ); \ + Ca ^= E##ga; \ + E##ge = Bge ^( Bgi & Bgo ); \ + Ce ^= E##ge; \ + E##gi = Bgi ^( Bgo |(~Bgu)); \ + Ci ^= E##gi; \ + E##go = Bgo ^( Bgu | Bga ); \ + Co ^= E##go; \ + E##gu = Bgu ^( Bga & Bge ); \ + Cu ^= E##gu; \ +\ + A##be ^= De; \ + Bka = ROL64(A##be, 1); \ + A##gi ^= Di; \ + Bke = ROL64(A##gi, 6); \ + A##ko ^= Do; \ + Bki = ROL64(A##ko, 25); \ + A##mu ^= Du; \ + Bko = ROL64(A##mu, 8); \ + A##sa ^= Da; \ + Bku = ROL64(A##sa, 18); \ + E##ka = Bka ^( Bke | Bki ); \ + Ca ^= E##ka; \ + E##ke = Bke ^( Bki & Bko ); \ + Ce ^= E##ke; \ + E##ki = Bki ^((~Bko)& Bku ); \ + Ci ^= E##ki; \ + E##ko = (~Bko)^( Bku | Bka ); \ + Co ^= E##ko; \ + E##ku = Bku ^( Bka & Bke ); \ + Cu ^= E##ku; \ +\ + A##bu ^= Du; \ + Bma = ROL64(A##bu, 27); \ + A##ga ^= Da; \ + Bme = ROL64(A##ga, 36); \ + A##ke ^= De; \ + Bmi = ROL64(A##ke, 10); \ + A##mi ^= Di; \ + Bmo = ROL64(A##mi, 15); \ + A##so ^= Do; \ + Bmu = ROL64(A##so, 56); \ + E##ma = Bma ^( Bme & Bmi ); \ + Ca ^= E##ma; \ + E##me = Bme ^( Bmi | Bmo ); \ + Ce ^= E##me; \ + E##mi = Bmi ^((~Bmo)| Bmu ); \ + Ci ^= E##mi; \ + E##mo = (~Bmo)^( Bmu & Bma ); \ + Co ^= E##mo; \ + E##mu = Bmu ^( Bma | Bme ); \ + Cu ^= E##mu; \ +\ + A##bi ^= Di; \ + Bsa = ROL64(A##bi, 62); \ + A##go ^= Do; \ + Bse = ROL64(A##go, 55); \ + A##ku ^= Du; \ + Bsi = ROL64(A##ku, 39); \ + A##ma ^= Da; \ + Bso = ROL64(A##ma, 41); \ + A##se ^= De; \ + Bsu = ROL64(A##se, 2); \ + E##sa = Bsa ^((~Bse)& Bsi ); \ + Ca ^= E##sa; \ + E##se = (~Bse)^( Bsi | Bso ); \ + Ce ^= E##se; \ + E##si = Bsi ^( Bso & Bsu ); \ + Ci ^= E##si; \ + E##so = Bso ^( Bsu | Bsa ); \ + Co ^= E##so; \ + E##su = Bsu ^( Bsa & Bse ); \ + Cu ^= E##su; \ +\ + +/* --- Code for round (lane complementing pattern 'bebigokimisa') */ + +/* --- 64-bit lanes mapped to 64-bit words */ + +#define thetaRhoPiChiIota(i, A, E) \ + Da = Cu^ROL64(Ce, 1); \ + De = Ca^ROL64(Ci, 1); \ + Di = Ce^ROL64(Co, 1); \ + Do = Ci^ROL64(Cu, 1); \ + Du = Co^ROL64(Ca, 1); \ +\ + A##ba ^= Da; \ + Bba = A##ba; \ + A##ge ^= De; \ + Bbe = ROL64(A##ge, 44); \ + A##ki ^= Di; \ + Bbi = ROL64(A##ki, 43); \ + A##mo ^= Do; \ + Bbo = ROL64(A##mo, 21); \ + A##su ^= Du; \ + Bbu = ROL64(A##su, 14); \ + E##ba = Bba ^( Bbe | Bbi ); \ + E##ba ^= KeccakF1600RoundConstants[i]; \ + E##be = Bbe ^((~Bbi)| Bbo ); \ + E##bi = Bbi ^( Bbo & Bbu ); \ + E##bo = Bbo ^( Bbu | Bba ); \ + E##bu = Bbu ^( Bba & Bbe ); \ +\ + A##bo ^= Do; \ + Bga = ROL64(A##bo, 28); \ + A##gu ^= Du; \ + Bge = ROL64(A##gu, 20); \ + A##ka ^= Da; \ + Bgi = ROL64(A##ka, 3); \ + A##me ^= De; \ + Bgo = ROL64(A##me, 45); \ + A##si ^= Di; \ + Bgu = ROL64(A##si, 61); \ + E##ga = Bga ^( Bge | Bgi ); \ + E##ge = Bge ^( Bgi & Bgo ); \ + E##gi = Bgi ^( Bgo |(~Bgu)); \ + E##go = Bgo ^( Bgu | Bga ); \ + E##gu = Bgu ^( Bga & Bge ); \ +\ + A##be ^= De; \ + Bka = ROL64(A##be, 1); \ + A##gi ^= Di; \ + Bke = ROL64(A##gi, 6); \ + A##ko ^= Do; \ + Bki = ROL64(A##ko, 25); \ + A##mu ^= Du; \ + Bko = ROL64(A##mu, 8); \ + A##sa ^= Da; \ + Bku = ROL64(A##sa, 18); \ + E##ka = Bka ^( Bke | Bki ); \ + E##ke = Bke ^( Bki & Bko ); \ + E##ki = Bki ^((~Bko)& Bku ); \ + E##ko = (~Bko)^( Bku | Bka ); \ + E##ku = Bku ^( Bka & Bke ); \ +\ + A##bu ^= Du; \ + Bma = ROL64(A##bu, 27); \ + A##ga ^= Da; \ + Bme = ROL64(A##ga, 36); \ + A##ke ^= De; \ + Bmi = ROL64(A##ke, 10); \ + A##mi ^= Di; \ + Bmo = ROL64(A##mi, 15); \ + A##so ^= Do; \ + Bmu = ROL64(A##so, 56); \ + E##ma = Bma ^( Bme & Bmi ); \ + E##me = Bme ^( Bmi | Bmo ); \ + E##mi = Bmi ^((~Bmo)| Bmu ); \ + E##mo = (~Bmo)^( Bmu & Bma ); \ + E##mu = Bmu ^( Bma | Bme ); \ +\ + A##bi ^= Di; \ + Bsa = ROL64(A##bi, 62); \ + A##go ^= Do; \ + Bse = ROL64(A##go, 55); \ + A##ku ^= Du; \ + Bsi = ROL64(A##ku, 39); \ + A##ma ^= Da; \ + Bso = ROL64(A##ma, 41); \ + A##se ^= De; \ + Bsu = ROL64(A##se, 2); \ + E##sa = Bsa ^((~Bse)& Bsi ); \ + E##se = (~Bse)^( Bsi | Bso ); \ + E##si = Bsi ^( Bso & Bsu ); \ + E##so = Bso ^( Bsu | Bsa ); \ + E##su = Bsu ^( Bsa & Bse ); \ +\ + +#else /* UseBebigokimisa */ + +/* --- Code for round, with prepare-theta */ + +/* --- 64-bit lanes mapped to 64-bit words */ + +#define thetaRhoPiChiIotaPrepareTheta(i, A, E) \ + Da = Cu^ROL64(Ce, 1); \ + De = Ca^ROL64(Ci, 1); \ + Di = Ce^ROL64(Co, 1); \ + Do = Ci^ROL64(Cu, 1); \ + Du = Co^ROL64(Ca, 1); \ +\ + A##ba ^= Da; \ + Bba = A##ba; \ + A##ge ^= De; \ + Bbe = ROL64(A##ge, 44); \ + A##ki ^= Di; \ + Bbi = ROL64(A##ki, 43); \ + A##mo ^= Do; \ + Bbo = ROL64(A##mo, 21); \ + A##su ^= Du; \ + Bbu = ROL64(A##su, 14); \ + E##ba = Bba ^((~Bbe)& Bbi ); \ + E##ba ^= KeccakF1600RoundConstants[i]; \ + Ca = E##ba; \ + E##be = Bbe ^((~Bbi)& Bbo ); \ + Ce = E##be; \ + E##bi = Bbi ^((~Bbo)& Bbu ); \ + Ci = E##bi; \ + E##bo = Bbo ^((~Bbu)& Bba ); \ + Co = E##bo; \ + E##bu = Bbu ^((~Bba)& Bbe ); \ + Cu = E##bu; \ +\ + A##bo ^= Do; \ + Bga = ROL64(A##bo, 28); \ + A##gu ^= Du; \ + Bge = ROL64(A##gu, 20); \ + A##ka ^= Da; \ + Bgi = ROL64(A##ka, 3); \ + A##me ^= De; \ + Bgo = ROL64(A##me, 45); \ + A##si ^= Di; \ + Bgu = ROL64(A##si, 61); \ + E##ga = Bga ^((~Bge)& Bgi ); \ + Ca ^= E##ga; \ + E##ge = Bge ^((~Bgi)& Bgo ); \ + Ce ^= E##ge; \ + E##gi = Bgi ^((~Bgo)& Bgu ); \ + Ci ^= E##gi; \ + E##go = Bgo ^((~Bgu)& Bga ); \ + Co ^= E##go; \ + E##gu = Bgu ^((~Bga)& Bge ); \ + Cu ^= E##gu; \ +\ + A##be ^= De; \ + Bka = ROL64(A##be, 1); \ + A##gi ^= Di; \ + Bke = ROL64(A##gi, 6); \ + A##ko ^= Do; \ + Bki = ROL64(A##ko, 25); \ + A##mu ^= Du; \ + Bko = ROL64(A##mu, 8); \ + A##sa ^= Da; \ + Bku = ROL64(A##sa, 18); \ + E##ka = Bka ^((~Bke)& Bki ); \ + Ca ^= E##ka; \ + E##ke = Bke ^((~Bki)& Bko ); \ + Ce ^= E##ke; \ + E##ki = Bki ^((~Bko)& Bku ); \ + Ci ^= E##ki; \ + E##ko = Bko ^((~Bku)& Bka ); \ + Co ^= E##ko; \ + E##ku = Bku ^((~Bka)& Bke ); \ + Cu ^= E##ku; \ +\ + A##bu ^= Du; \ + Bma = ROL64(A##bu, 27); \ + A##ga ^= Da; \ + Bme = ROL64(A##ga, 36); \ + A##ke ^= De; \ + Bmi = ROL64(A##ke, 10); \ + A##mi ^= Di; \ + Bmo = ROL64(A##mi, 15); \ + A##so ^= Do; \ + Bmu = ROL64(A##so, 56); \ + E##ma = Bma ^((~Bme)& Bmi ); \ + Ca ^= E##ma; \ + E##me = Bme ^((~Bmi)& Bmo ); \ + Ce ^= E##me; \ + E##mi = Bmi ^((~Bmo)& Bmu ); \ + Ci ^= E##mi; \ + E##mo = Bmo ^((~Bmu)& Bma ); \ + Co ^= E##mo; \ + E##mu = Bmu ^((~Bma)& Bme ); \ + Cu ^= E##mu; \ +\ + A##bi ^= Di; \ + Bsa = ROL64(A##bi, 62); \ + A##go ^= Do; \ + Bse = ROL64(A##go, 55); \ + A##ku ^= Du; \ + Bsi = ROL64(A##ku, 39); \ + A##ma ^= Da; \ + Bso = ROL64(A##ma, 41); \ + A##se ^= De; \ + Bsu = ROL64(A##se, 2); \ + E##sa = Bsa ^((~Bse)& Bsi ); \ + Ca ^= E##sa; \ + E##se = Bse ^((~Bsi)& Bso ); \ + Ce ^= E##se; \ + E##si = Bsi ^((~Bso)& Bsu ); \ + Ci ^= E##si; \ + E##so = Bso ^((~Bsu)& Bsa ); \ + Co ^= E##so; \ + E##su = Bsu ^((~Bsa)& Bse ); \ + Cu ^= E##su; \ +\ + +/* --- Code for round */ + +/* --- 64-bit lanes mapped to 64-bit words */ + +#define thetaRhoPiChiIota(i, A, E) \ + Da = Cu^ROL64(Ce, 1); \ + De = Ca^ROL64(Ci, 1); \ + Di = Ce^ROL64(Co, 1); \ + Do = Ci^ROL64(Cu, 1); \ + Du = Co^ROL64(Ca, 1); \ +\ + A##ba ^= Da; \ + Bba = A##ba; \ + A##ge ^= De; \ + Bbe = ROL64(A##ge, 44); \ + A##ki ^= Di; \ + Bbi = ROL64(A##ki, 43); \ + A##mo ^= Do; \ + Bbo = ROL64(A##mo, 21); \ + A##su ^= Du; \ + Bbu = ROL64(A##su, 14); \ + E##ba = Bba ^((~Bbe)& Bbi ); \ + E##ba ^= KeccakF1600RoundConstants[i]; \ + E##be = Bbe ^((~Bbi)& Bbo ); \ + E##bi = Bbi ^((~Bbo)& Bbu ); \ + E##bo = Bbo ^((~Bbu)& Bba ); \ + E##bu = Bbu ^((~Bba)& Bbe ); \ +\ + A##bo ^= Do; \ + Bga = ROL64(A##bo, 28); \ + A##gu ^= Du; \ + Bge = ROL64(A##gu, 20); \ + A##ka ^= Da; \ + Bgi = ROL64(A##ka, 3); \ + A##me ^= De; \ + Bgo = ROL64(A##me, 45); \ + A##si ^= Di; \ + Bgu = ROL64(A##si, 61); \ + E##ga = Bga ^((~Bge)& Bgi ); \ + E##ge = Bge ^((~Bgi)& Bgo ); \ + E##gi = Bgi ^((~Bgo)& Bgu ); \ + E##go = Bgo ^((~Bgu)& Bga ); \ + E##gu = Bgu ^((~Bga)& Bge ); \ +\ + A##be ^= De; \ + Bka = ROL64(A##be, 1); \ + A##gi ^= Di; \ + Bke = ROL64(A##gi, 6); \ + A##ko ^= Do; \ + Bki = ROL64(A##ko, 25); \ + A##mu ^= Du; \ + Bko = ROL64(A##mu, 8); \ + A##sa ^= Da; \ + Bku = ROL64(A##sa, 18); \ + E##ka = Bka ^((~Bke)& Bki ); \ + E##ke = Bke ^((~Bki)& Bko ); \ + E##ki = Bki ^((~Bko)& Bku ); \ + E##ko = Bko ^((~Bku)& Bka ); \ + E##ku = Bku ^((~Bka)& Bke ); \ +\ + A##bu ^= Du; \ + Bma = ROL64(A##bu, 27); \ + A##ga ^= Da; \ + Bme = ROL64(A##ga, 36); \ + A##ke ^= De; \ + Bmi = ROL64(A##ke, 10); \ + A##mi ^= Di; \ + Bmo = ROL64(A##mi, 15); \ + A##so ^= Do; \ + Bmu = ROL64(A##so, 56); \ + E##ma = Bma ^((~Bme)& Bmi ); \ + E##me = Bme ^((~Bmi)& Bmo ); \ + E##mi = Bmi ^((~Bmo)& Bmu ); \ + E##mo = Bmo ^((~Bmu)& Bma ); \ + E##mu = Bmu ^((~Bma)& Bme ); \ +\ + A##bi ^= Di; \ + Bsa = ROL64(A##bi, 62); \ + A##go ^= Do; \ + Bse = ROL64(A##go, 55); \ + A##ku ^= Du; \ + Bsi = ROL64(A##ku, 39); \ + A##ma ^= Da; \ + Bso = ROL64(A##ma, 41); \ + A##se ^= De; \ + Bsu = ROL64(A##se, 2); \ + E##sa = Bsa ^((~Bse)& Bsi ); \ + E##se = Bse ^((~Bsi)& Bso ); \ + E##si = Bsi ^((~Bso)& Bsu ); \ + E##so = Bso ^((~Bsu)& Bsa ); \ + E##su = Bsu ^((~Bsa)& Bse ); \ +\ + +#endif /* UseBebigokimisa */ + + +#define copyFromState(X, state) \ + X##ba = state[ 0]; \ + X##be = state[ 1]; \ + X##bi = state[ 2]; \ + X##bo = state[ 3]; \ + X##bu = state[ 4]; \ + X##ga = state[ 5]; \ + X##ge = state[ 6]; \ + X##gi = state[ 7]; \ + X##go = state[ 8]; \ + X##gu = state[ 9]; \ + X##ka = state[10]; \ + X##ke = state[11]; \ + X##ki = state[12]; \ + X##ko = state[13]; \ + X##ku = state[14]; \ + X##ma = state[15]; \ + X##me = state[16]; \ + X##mi = state[17]; \ + X##mo = state[18]; \ + X##mu = state[19]; \ + X##sa = state[20]; \ + X##se = state[21]; \ + X##si = state[22]; \ + X##so = state[23]; \ + X##su = state[24]; \ + +#define copyToState(state, X) \ + state[ 0] = X##ba; \ + state[ 1] = X##be; \ + state[ 2] = X##bi; \ + state[ 3] = X##bo; \ + state[ 4] = X##bu; \ + state[ 5] = X##ga; \ + state[ 6] = X##ge; \ + state[ 7] = X##gi; \ + state[ 8] = X##go; \ + state[ 9] = X##gu; \ + state[10] = X##ka; \ + state[11] = X##ke; \ + state[12] = X##ki; \ + state[13] = X##ko; \ + state[14] = X##ku; \ + state[15] = X##ma; \ + state[16] = X##me; \ + state[17] = X##mi; \ + state[18] = X##mo; \ + state[19] = X##mu; \ + state[20] = X##sa; \ + state[21] = X##se; \ + state[22] = X##si; \ + state[23] = X##so; \ + state[24] = X##su; \ + +#define copyStateVariables(X, Y) \ + X##ba = Y##ba; \ + X##be = Y##be; \ + X##bi = Y##bi; \ + X##bo = Y##bo; \ + X##bu = Y##bu; \ + X##ga = Y##ga; \ + X##ge = Y##ge; \ + X##gi = Y##gi; \ + X##go = Y##go; \ + X##gu = Y##gu; \ + X##ka = Y##ka; \ + X##ke = Y##ke; \ + X##ki = Y##ki; \ + X##ko = Y##ko; \ + X##ku = Y##ku; \ + X##ma = Y##ma; \ + X##me = Y##me; \ + X##mi = Y##mi; \ + X##mo = Y##mo; \ + X##mu = Y##mu; \ + X##sa = Y##sa; \ + X##se = Y##se; \ + X##si = Y##si; \ + X##so = Y##so; \ + X##su = Y##su; \ + +#define copyFromStateAndAdd(X, state, input, laneCount) \ + if (laneCount < 16) { \ + if (laneCount < 8) { \ + if (laneCount < 4) { \ + if (laneCount < 2) { \ + if (laneCount < 1) { \ + X##ba = state[ 0]; \ + } \ + else { \ + X##ba = state[ 0]^input[ 0]; \ + } \ + X##be = state[ 1]; \ + X##bi = state[ 2]; \ + } \ + else { \ + X##ba = state[ 0]^input[ 0]; \ + X##be = state[ 1]^input[ 1]; \ + if (laneCount < 3) { \ + X##bi = state[ 2]; \ + } \ + else { \ + X##bi = state[ 2]^input[ 2]; \ + } \ + } \ + X##bo = state[ 3]; \ + X##bu = state[ 4]; \ + X##ga = state[ 5]; \ + X##ge = state[ 6]; \ + } \ + else { \ + X##ba = state[ 0]^input[ 0]; \ + X##be = state[ 1]^input[ 1]; \ + X##bi = state[ 2]^input[ 2]; \ + X##bo = state[ 3]^input[ 3]; \ + if (laneCount < 6) { \ + if (laneCount < 5) { \ + X##bu = state[ 4]; \ + } \ + else { \ + X##bu = state[ 4]^input[ 4]; \ + } \ + X##ga = state[ 5]; \ + X##ge = state[ 6]; \ + } \ + else { \ + X##bu = state[ 4]^input[ 4]; \ + X##ga = state[ 5]^input[ 5]; \ + if (laneCount < 7) { \ + X##ge = state[ 6]; \ + } \ + else { \ + X##ge = state[ 6]^input[ 6]; \ + } \ + } \ + } \ + X##gi = state[ 7]; \ + X##go = state[ 8]; \ + X##gu = state[ 9]; \ + X##ka = state[10]; \ + X##ke = state[11]; \ + X##ki = state[12]; \ + X##ko = state[13]; \ + X##ku = state[14]; \ + } \ + else { \ + X##ba = state[ 0]^input[ 0]; \ + X##be = state[ 1]^input[ 1]; \ + X##bi = state[ 2]^input[ 2]; \ + X##bo = state[ 3]^input[ 3]; \ + X##bu = state[ 4]^input[ 4]; \ + X##ga = state[ 5]^input[ 5]; \ + X##ge = state[ 6]^input[ 6]; \ + X##gi = state[ 7]^input[ 7]; \ + if (laneCount < 12) { \ + if (laneCount < 10) { \ + if (laneCount < 9) { \ + X##go = state[ 8]; \ + } \ + else { \ + X##go = state[ 8]^input[ 8]; \ + } \ + X##gu = state[ 9]; \ + X##ka = state[10]; \ + } \ + else { \ + X##go = state[ 8]^input[ 8]; \ + X##gu = state[ 9]^input[ 9]; \ + if (laneCount < 11) { \ + X##ka = state[10]; \ + } \ + else { \ + X##ka = state[10]^input[10]; \ + } \ + } \ + X##ke = state[11]; \ + X##ki = state[12]; \ + X##ko = state[13]; \ + X##ku = state[14]; \ + } \ + else { \ + X##go = state[ 8]^input[ 8]; \ + X##gu = state[ 9]^input[ 9]; \ + X##ka = state[10]^input[10]; \ + X##ke = state[11]^input[11]; \ + if (laneCount < 14) { \ + if (laneCount < 13) { \ + X##ki = state[12]; \ + } \ + else { \ + X##ki = state[12]^input[12]; \ + } \ + X##ko = state[13]; \ + X##ku = state[14]; \ + } \ + else { \ + X##ki = state[12]^input[12]; \ + X##ko = state[13]^input[13]; \ + if (laneCount < 15) { \ + X##ku = state[14]; \ + } \ + else { \ + X##ku = state[14]^input[14]; \ + } \ + } \ + } \ + } \ + X##ma = state[15]; \ + X##me = state[16]; \ + X##mi = state[17]; \ + X##mo = state[18]; \ + X##mu = state[19]; \ + X##sa = state[20]; \ + X##se = state[21]; \ + X##si = state[22]; \ + X##so = state[23]; \ + X##su = state[24]; \ + } \ + else { \ + X##ba = state[ 0]^input[ 0]; \ + X##be = state[ 1]^input[ 1]; \ + X##bi = state[ 2]^input[ 2]; \ + X##bo = state[ 3]^input[ 3]; \ + X##bu = state[ 4]^input[ 4]; \ + X##ga = state[ 5]^input[ 5]; \ + X##ge = state[ 6]^input[ 6]; \ + X##gi = state[ 7]^input[ 7]; \ + X##go = state[ 8]^input[ 8]; \ + X##gu = state[ 9]^input[ 9]; \ + X##ka = state[10]^input[10]; \ + X##ke = state[11]^input[11]; \ + X##ki = state[12]^input[12]; \ + X##ko = state[13]^input[13]; \ + X##ku = state[14]^input[14]; \ + X##ma = state[15]^input[15]; \ + if (laneCount < 24) { \ + if (laneCount < 20) { \ + if (laneCount < 18) { \ + if (laneCount < 17) { \ + X##me = state[16]; \ + } \ + else { \ + X##me = state[16]^input[16]; \ + } \ + X##mi = state[17]; \ + X##mo = state[18]; \ + } \ + else { \ + X##me = state[16]^input[16]; \ + X##mi = state[17]^input[17]; \ + if (laneCount < 19) { \ + X##mo = state[18]; \ + } \ + else { \ + X##mo = state[18]^input[18]; \ + } \ + } \ + X##mu = state[19]; \ + X##sa = state[20]; \ + X##se = state[21]; \ + X##si = state[22]; \ + } \ + else { \ + X##me = state[16]^input[16]; \ + X##mi = state[17]^input[17]; \ + X##mo = state[18]^input[18]; \ + X##mu = state[19]^input[19]; \ + if (laneCount < 22) { \ + if (laneCount < 21) { \ + X##sa = state[20]; \ + } \ + else { \ + X##sa = state[20]^input[20]; \ + } \ + X##se = state[21]; \ + X##si = state[22]; \ + } \ + else { \ + X##sa = state[20]^input[20]; \ + X##se = state[21]^input[21]; \ + if (laneCount < 23) { \ + X##si = state[22]; \ + } \ + else { \ + X##si = state[22]^input[22]; \ + } \ + } \ + } \ + X##so = state[23]; \ + X##su = state[24]; \ + } \ + else { \ + X##me = state[16]^input[16]; \ + X##mi = state[17]^input[17]; \ + X##mo = state[18]^input[18]; \ + X##mu = state[19]^input[19]; \ + X##sa = state[20]^input[20]; \ + X##se = state[21]^input[21]; \ + X##si = state[22]^input[22]; \ + X##so = state[23]^input[23]; \ + if (laneCount < 25) { \ + X##su = state[24]; \ + } \ + else { \ + X##su = state[24]^input[24]; \ + } \ + } \ + } + +#define addInput(X, input, laneCount) \ + if (laneCount == 21) { \ + X##ba ^= input[ 0]; \ + X##be ^= input[ 1]; \ + X##bi ^= input[ 2]; \ + X##bo ^= input[ 3]; \ + X##bu ^= input[ 4]; \ + X##ga ^= input[ 5]; \ + X##ge ^= input[ 6]; \ + X##gi ^= input[ 7]; \ + X##go ^= input[ 8]; \ + X##gu ^= input[ 9]; \ + X##ka ^= input[10]; \ + X##ke ^= input[11]; \ + X##ki ^= input[12]; \ + X##ko ^= input[13]; \ + X##ku ^= input[14]; \ + X##ma ^= input[15]; \ + X##me ^= input[16]; \ + X##mi ^= input[17]; \ + X##mo ^= input[18]; \ + X##mu ^= input[19]; \ + X##sa ^= input[20]; \ + } \ + else if (laneCount < 16) { \ + if (laneCount < 8) { \ + if (laneCount < 4) { \ + if (laneCount < 2) { \ + if (laneCount < 1) { \ + } \ + else { \ + X##ba ^= input[ 0]; \ + } \ + } \ + else { \ + X##ba ^= input[ 0]; \ + X##be ^= input[ 1]; \ + if (laneCount < 3) { \ + } \ + else { \ + X##bi ^= input[ 2]; \ + } \ + } \ + } \ + else { \ + X##ba ^= input[ 0]; \ + X##be ^= input[ 1]; \ + X##bi ^= input[ 2]; \ + X##bo ^= input[ 3]; \ + if (laneCount < 6) { \ + if (laneCount < 5) { \ + } \ + else { \ + X##bu ^= input[ 4]; \ + } \ + } \ + else { \ + X##bu ^= input[ 4]; \ + X##ga ^= input[ 5]; \ + if (laneCount < 7) { \ + } \ + else { \ + X##ge ^= input[ 6]; \ + } \ + } \ + } \ + } \ + else { \ + X##ba ^= input[ 0]; \ + X##be ^= input[ 1]; \ + X##bi ^= input[ 2]; \ + X##bo ^= input[ 3]; \ + X##bu ^= input[ 4]; \ + X##ga ^= input[ 5]; \ + X##ge ^= input[ 6]; \ + X##gi ^= input[ 7]; \ + if (laneCount < 12) { \ + if (laneCount < 10) { \ + if (laneCount < 9) { \ + } \ + else { \ + X##go ^= input[ 8]; \ + } \ + } \ + else { \ + X##go ^= input[ 8]; \ + X##gu ^= input[ 9]; \ + if (laneCount < 11) { \ + } \ + else { \ + X##ka ^= input[10]; \ + } \ + } \ + } \ + else { \ + X##go ^= input[ 8]; \ + X##gu ^= input[ 9]; \ + X##ka ^= input[10]; \ + X##ke ^= input[11]; \ + if (laneCount < 14) { \ + if (laneCount < 13) { \ + } \ + else { \ + X##ki ^= input[12]; \ + } \ + } \ + else { \ + X##ki ^= input[12]; \ + X##ko ^= input[13]; \ + if (laneCount < 15) { \ + } \ + else { \ + X##ku ^= input[14]; \ + } \ + } \ + } \ + } \ + } \ + else { \ + X##ba ^= input[ 0]; \ + X##be ^= input[ 1]; \ + X##bi ^= input[ 2]; \ + X##bo ^= input[ 3]; \ + X##bu ^= input[ 4]; \ + X##ga ^= input[ 5]; \ + X##ge ^= input[ 6]; \ + X##gi ^= input[ 7]; \ + X##go ^= input[ 8]; \ + X##gu ^= input[ 9]; \ + X##ka ^= input[10]; \ + X##ke ^= input[11]; \ + X##ki ^= input[12]; \ + X##ko ^= input[13]; \ + X##ku ^= input[14]; \ + X##ma ^= input[15]; \ + if (laneCount < 24) { \ + if (laneCount < 20) { \ + if (laneCount < 18) { \ + if (laneCount < 17) { \ + } \ + else { \ + X##me ^= input[16]; \ + } \ + } \ + else { \ + X##me ^= input[16]; \ + X##mi ^= input[17]; \ + if (laneCount < 19) { \ + } \ + else { \ + X##mo ^= input[18]; \ + } \ + } \ + } \ + else { \ + X##me ^= input[16]; \ + X##mi ^= input[17]; \ + X##mo ^= input[18]; \ + X##mu ^= input[19]; \ + if (laneCount < 22) { \ + if (laneCount < 21) { \ + } \ + else { \ + X##sa ^= input[20]; \ + } \ + } \ + else { \ + X##sa ^= input[20]; \ + X##se ^= input[21]; \ + if (laneCount < 23) { \ + } \ + else { \ + X##si ^= input[22]; \ + } \ + } \ + } \ + } \ + else { \ + X##me ^= input[16]; \ + X##mi ^= input[17]; \ + X##mo ^= input[18]; \ + X##mu ^= input[19]; \ + X##sa ^= input[20]; \ + X##se ^= input[21]; \ + X##si ^= input[22]; \ + X##so ^= input[23]; \ + if (laneCount < 25) { \ + } \ + else { \ + X##su ^= input[24]; \ + } \ + } \ + } + +#ifdef UseBebigokimisa + +#define copyToStateAndOutput(X, state, output, laneCount) \ + if (laneCount < 16) { \ + if (laneCount < 8) { \ + if (laneCount < 4) { \ + if (laneCount < 2) { \ + state[ 0] = X##ba; \ + if (laneCount >= 1) { \ + output[ 0] = X##ba; \ + } \ + state[ 1] = X##be; \ + state[ 2] = X##bi; \ + } \ + else { \ + state[ 0] = X##ba; \ + output[ 0] = X##ba; \ + state[ 1] = X##be; \ + output[ 1] = ~X##be; \ + state[ 2] = X##bi; \ + if (laneCount >= 3) { \ + output[ 2] = ~X##bi; \ + } \ + } \ + state[ 3] = X##bo; \ + state[ 4] = X##bu; \ + state[ 5] = X##ga; \ + state[ 6] = X##ge; \ + } \ + else { \ + state[ 0] = X##ba; \ + output[ 0] = X##ba; \ + state[ 1] = X##be; \ + output[ 1] = ~X##be; \ + state[ 2] = X##bi; \ + output[ 2] = ~X##bi; \ + state[ 3] = X##bo; \ + output[ 3] = X##bo; \ + if (laneCount < 6) { \ + state[ 4] = X##bu; \ + if (laneCount >= 5) { \ + output[ 4] = X##bu; \ + } \ + state[ 5] = X##ga; \ + state[ 6] = X##ge; \ + } \ + else { \ + state[ 4] = X##bu; \ + output[ 4] = X##bu; \ + state[ 5] = X##ga; \ + output[ 5] = X##ga; \ + state[ 6] = X##ge; \ + if (laneCount >= 7) { \ + output[ 6] = X##ge; \ + } \ + } \ + } \ + state[ 7] = X##gi; \ + state[ 8] = X##go; \ + state[ 9] = X##gu; \ + state[10] = X##ka; \ + state[11] = X##ke; \ + state[12] = X##ki; \ + state[13] = X##ko; \ + state[14] = X##ku; \ + } \ + else { \ + state[ 0] = X##ba; \ + output[ 0] = X##ba; \ + state[ 1] = X##be; \ + output[ 1] = ~X##be; \ + state[ 2] = X##bi; \ + output[ 2] = ~X##bi; \ + state[ 3] = X##bo; \ + output[ 3] = X##bo; \ + state[ 4] = X##bu; \ + output[ 4] = X##bu; \ + state[ 5] = X##ga; \ + output[ 5] = X##ga; \ + state[ 6] = X##ge; \ + output[ 6] = X##ge; \ + state[ 7] = X##gi; \ + output[ 7] = X##gi; \ + if (laneCount < 12) { \ + if (laneCount < 10) { \ + state[ 8] = X##go; \ + if (laneCount >= 9) { \ + output[ 8] = ~X##go; \ + } \ + state[ 9] = X##gu; \ + state[10] = X##ka; \ + } \ + else { \ + state[ 8] = X##go; \ + output[ 8] = ~X##go; \ + state[ 9] = X##gu; \ + output[ 9] = X##gu; \ + state[10] = X##ka; \ + if (laneCount >= 11) { \ + output[10] = X##ka; \ + } \ + } \ + state[11] = X##ke; \ + state[12] = X##ki; \ + state[13] = X##ko; \ + state[14] = X##ku; \ + } \ + else { \ + state[ 8] = X##go; \ + output[ 8] = ~X##go; \ + state[ 9] = X##gu; \ + output[ 9] = X##gu; \ + state[10] = X##ka; \ + output[10] = X##ka; \ + state[11] = X##ke; \ + output[11] = X##ke; \ + if (laneCount < 14) { \ + state[12] = X##ki; \ + if (laneCount >= 13) { \ + output[12] = ~X##ki; \ + } \ + state[13] = X##ko; \ + state[14] = X##ku; \ + } \ + else { \ + state[12] = X##ki; \ + output[12] = ~X##ki; \ + state[13] = X##ko; \ + output[13] = X##ko; \ + state[14] = X##ku; \ + if (laneCount >= 15) { \ + output[14] = X##ku; \ + } \ + } \ + } \ + } \ + state[15] = X##ma; \ + state[16] = X##me; \ + state[17] = X##mi; \ + state[18] = X##mo; \ + state[19] = X##mu; \ + state[20] = X##sa; \ + state[21] = X##se; \ + state[22] = X##si; \ + state[23] = X##so; \ + state[24] = X##su; \ + } \ + else { \ + state[ 0] = X##ba; \ + output[ 0] = X##ba; \ + state[ 1] = X##be; \ + output[ 1] = ~X##be; \ + state[ 2] = X##bi; \ + output[ 2] = ~X##bi; \ + state[ 3] = X##bo; \ + output[ 3] = X##bo; \ + state[ 4] = X##bu; \ + output[ 4] = X##bu; \ + state[ 5] = X##ga; \ + output[ 5] = X##ga; \ + state[ 6] = X##ge; \ + output[ 6] = X##ge; \ + state[ 7] = X##gi; \ + output[ 7] = X##gi; \ + state[ 8] = X##go; \ + output[ 8] = ~X##go; \ + state[ 9] = X##gu; \ + output[ 9] = X##gu; \ + state[10] = X##ka; \ + output[10] = X##ka; \ + state[11] = X##ke; \ + output[11] = X##ke; \ + state[12] = X##ki; \ + output[12] = ~X##ki; \ + state[13] = X##ko; \ + output[13] = X##ko; \ + state[14] = X##ku; \ + output[14] = X##ku; \ + state[15] = X##ma; \ + output[15] = X##ma; \ + if (laneCount < 24) { \ + if (laneCount < 20) { \ + if (laneCount < 18) { \ + state[16] = X##me; \ + if (laneCount >= 17) { \ + output[16] = X##me; \ + } \ + state[17] = X##mi; \ + state[18] = X##mo; \ + } \ + else { \ + state[16] = X##me; \ + output[16] = X##me; \ + state[17] = X##mi; \ + output[17] = ~X##mi; \ + state[18] = X##mo; \ + if (laneCount >= 19) { \ + output[18] = X##mo; \ + } \ + } \ + state[19] = X##mu; \ + state[20] = X##sa; \ + state[21] = X##se; \ + state[22] = X##si; \ + } \ + else { \ + state[16] = X##me; \ + output[16] = X##me; \ + state[17] = X##mi; \ + output[17] = ~X##mi; \ + state[18] = X##mo; \ + output[18] = X##mo; \ + state[19] = X##mu; \ + output[19] = X##mu; \ + if (laneCount < 22) { \ + state[20] = X##sa; \ + if (laneCount >= 21) { \ + output[20] = ~X##sa; \ + } \ + state[21] = X##se; \ + state[22] = X##si; \ + } \ + else { \ + state[20] = X##sa; \ + output[20] = ~X##sa; \ + state[21] = X##se; \ + output[21] = X##se; \ + state[22] = X##si; \ + if (laneCount >= 23) { \ + output[22] = X##si; \ + } \ + } \ + } \ + state[23] = X##so; \ + state[24] = X##su; \ + } \ + else { \ + state[16] = X##me; \ + output[16] = X##me; \ + state[17] = X##mi; \ + output[17] = ~X##mi; \ + state[18] = X##mo; \ + output[18] = X##mo; \ + state[19] = X##mu; \ + output[19] = X##mu; \ + state[20] = X##sa; \ + output[20] = ~X##sa; \ + state[21] = X##se; \ + output[21] = X##se; \ + state[22] = X##si; \ + output[22] = X##si; \ + state[23] = X##so; \ + output[23] = X##so; \ + state[24] = X##su; \ + if (laneCount >= 25) { \ + output[24] = X##su; \ + } \ + } \ + } + +#define output(X, output, laneCount) \ + if (laneCount < 16) { \ + if (laneCount < 8) { \ + if (laneCount < 4) { \ + if (laneCount < 2) { \ + if (laneCount >= 1) { \ + output[ 0] = X##ba; \ + } \ + } \ + else { \ + output[ 0] = X##ba; \ + output[ 1] = ~X##be; \ + if (laneCount >= 3) { \ + output[ 2] = ~X##bi; \ + } \ + } \ + } \ + else { \ + output[ 0] = X##ba; \ + output[ 1] = ~X##be; \ + output[ 2] = ~X##bi; \ + output[ 3] = X##bo; \ + if (laneCount < 6) { \ + if (laneCount >= 5) { \ + output[ 4] = X##bu; \ + } \ + } \ + else { \ + output[ 4] = X##bu; \ + output[ 5] = X##ga; \ + if (laneCount >= 7) { \ + output[ 6] = X##ge; \ + } \ + } \ + } \ + } \ + else { \ + output[ 0] = X##ba; \ + output[ 1] = ~X##be; \ + output[ 2] = ~X##bi; \ + output[ 3] = X##bo; \ + output[ 4] = X##bu; \ + output[ 5] = X##ga; \ + output[ 6] = X##ge; \ + output[ 7] = X##gi; \ + if (laneCount < 12) { \ + if (laneCount < 10) { \ + if (laneCount >= 9) { \ + output[ 8] = ~X##go; \ + } \ + } \ + else { \ + output[ 8] = ~X##go; \ + output[ 9] = X##gu; \ + if (laneCount >= 11) { \ + output[10] = X##ka; \ + } \ + } \ + } \ + else { \ + output[ 8] = ~X##go; \ + output[ 9] = X##gu; \ + output[10] = X##ka; \ + output[11] = X##ke; \ + if (laneCount < 14) { \ + if (laneCount >= 13) { \ + output[12] = ~X##ki; \ + } \ + } \ + else { \ + output[12] = ~X##ki; \ + output[13] = X##ko; \ + if (laneCount >= 15) { \ + output[14] = X##ku; \ + } \ + } \ + } \ + } \ + } \ + else { \ + output[ 0] = X##ba; \ + output[ 1] = ~X##be; \ + output[ 2] = ~X##bi; \ + output[ 3] = X##bo; \ + output[ 4] = X##bu; \ + output[ 5] = X##ga; \ + output[ 6] = X##ge; \ + output[ 7] = X##gi; \ + output[ 8] = ~X##go; \ + output[ 9] = X##gu; \ + output[10] = X##ka; \ + output[11] = X##ke; \ + output[12] = ~X##ki; \ + output[13] = X##ko; \ + output[14] = X##ku; \ + output[15] = X##ma; \ + if (laneCount < 24) { \ + if (laneCount < 20) { \ + if (laneCount < 18) { \ + if (laneCount >= 17) { \ + output[16] = X##me; \ + } \ + } \ + else { \ + output[16] = X##me; \ + output[17] = ~X##mi; \ + if (laneCount >= 19) { \ + output[18] = X##mo; \ + } \ + } \ + } \ + else { \ + output[16] = X##me; \ + output[17] = ~X##mi; \ + output[18] = X##mo; \ + output[19] = X##mu; \ + if (laneCount < 22) { \ + if (laneCount >= 21) { \ + output[20] = ~X##sa; \ + } \ + } \ + else { \ + output[20] = ~X##sa; \ + output[21] = X##se; \ + if (laneCount >= 23) { \ + output[22] = X##si; \ + } \ + } \ + } \ + } \ + else { \ + output[16] = X##me; \ + output[17] = ~X##mi; \ + output[18] = X##mo; \ + output[19] = X##mu; \ + output[20] = ~X##sa; \ + output[21] = X##se; \ + output[22] = X##si; \ + output[23] = X##so; \ + if (laneCount >= 25) { \ + output[24] = X##su; \ + } \ + } \ + } + +#define wrapOne(X, input, output, index, name) \ + X##name ^= input[index]; \ + output[index] = X##name; + +#define wrapOneInvert(X, input, output, index, name) \ + X##name ^= input[index]; \ + output[index] = ~X##name; + +#define unwrapOne(X, input, output, index, name) \ + output[index] = input[index] ^ X##name; \ + X##name ^= output[index]; + +#define unwrapOneInvert(X, input, output, index, name) \ + output[index] = ~(input[index] ^ X##name); \ + X##name ^= output[index]; \ + +#else /* UseBebigokimisa */ + + +#define copyToStateAndOutput(X, state, output, laneCount) \ + if (laneCount < 16) { \ + if (laneCount < 8) { \ + if (laneCount < 4) { \ + if (laneCount < 2) { \ + state[ 0] = X##ba; \ + if (laneCount >= 1) { \ + output[ 0] = X##ba; \ + } \ + state[ 1] = X##be; \ + state[ 2] = X##bi; \ + } \ + else { \ + state[ 0] = X##ba; \ + output[ 0] = X##ba; \ + state[ 1] = X##be; \ + output[ 1] = X##be; \ + state[ 2] = X##bi; \ + if (laneCount >= 3) { \ + output[ 2] = X##bi; \ + } \ + } \ + state[ 3] = X##bo; \ + state[ 4] = X##bu; \ + state[ 5] = X##ga; \ + state[ 6] = X##ge; \ + } \ + else { \ + state[ 0] = X##ba; \ + output[ 0] = X##ba; \ + state[ 1] = X##be; \ + output[ 1] = X##be; \ + state[ 2] = X##bi; \ + output[ 2] = X##bi; \ + state[ 3] = X##bo; \ + output[ 3] = X##bo; \ + if (laneCount < 6) { \ + state[ 4] = X##bu; \ + if (laneCount >= 5) { \ + output[ 4] = X##bu; \ + } \ + state[ 5] = X##ga; \ + state[ 6] = X##ge; \ + } \ + else { \ + state[ 4] = X##bu; \ + output[ 4] = X##bu; \ + state[ 5] = X##ga; \ + output[ 5] = X##ga; \ + state[ 6] = X##ge; \ + if (laneCount >= 7) { \ + output[ 6] = X##ge; \ + } \ + } \ + } \ + state[ 7] = X##gi; \ + state[ 8] = X##go; \ + state[ 9] = X##gu; \ + state[10] = X##ka; \ + state[11] = X##ke; \ + state[12] = X##ki; \ + state[13] = X##ko; \ + state[14] = X##ku; \ + } \ + else { \ + state[ 0] = X##ba; \ + output[ 0] = X##ba; \ + state[ 1] = X##be; \ + output[ 1] = X##be; \ + state[ 2] = X##bi; \ + output[ 2] = X##bi; \ + state[ 3] = X##bo; \ + output[ 3] = X##bo; \ + state[ 4] = X##bu; \ + output[ 4] = X##bu; \ + state[ 5] = X##ga; \ + output[ 5] = X##ga; \ + state[ 6] = X##ge; \ + output[ 6] = X##ge; \ + state[ 7] = X##gi; \ + output[ 7] = X##gi; \ + if (laneCount < 12) { \ + if (laneCount < 10) { \ + state[ 8] = X##go; \ + if (laneCount >= 9) { \ + output[ 8] = X##go; \ + } \ + state[ 9] = X##gu; \ + state[10] = X##ka; \ + } \ + else { \ + state[ 8] = X##go; \ + output[ 8] = X##go; \ + state[ 9] = X##gu; \ + output[ 9] = X##gu; \ + state[10] = X##ka; \ + if (laneCount >= 11) { \ + output[10] = X##ka; \ + } \ + } \ + state[11] = X##ke; \ + state[12] = X##ki; \ + state[13] = X##ko; \ + state[14] = X##ku; \ + } \ + else { \ + state[ 8] = X##go; \ + output[ 8] = X##go; \ + state[ 9] = X##gu; \ + output[ 9] = X##gu; \ + state[10] = X##ka; \ + output[10] = X##ka; \ + state[11] = X##ke; \ + output[11] = X##ke; \ + if (laneCount < 14) { \ + state[12] = X##ki; \ + if (laneCount >= 13) { \ + output[12]= X##ki; \ + } \ + state[13] = X##ko; \ + state[14] = X##ku; \ + } \ + else { \ + state[12] = X##ki; \ + output[12]= X##ki; \ + state[13] = X##ko; \ + output[13] = X##ko; \ + state[14] = X##ku; \ + if (laneCount >= 15) { \ + output[14] = X##ku; \ + } \ + } \ + } \ + } \ + state[15] = X##ma; \ + state[16] = X##me; \ + state[17] = X##mi; \ + state[18] = X##mo; \ + state[19] = X##mu; \ + state[20] = X##sa; \ + state[21] = X##se; \ + state[22] = X##si; \ + state[23] = X##so; \ + state[24] = X##su; \ + } \ + else { \ + state[ 0] = X##ba; \ + output[ 0] = X##ba; \ + state[ 1] = X##be; \ + output[ 1] = X##be; \ + state[ 2] = X##bi; \ + output[ 2] = X##bi; \ + state[ 3] = X##bo; \ + output[ 3] = X##bo; \ + state[ 4] = X##bu; \ + output[ 4] = X##bu; \ + state[ 5] = X##ga; \ + output[ 5] = X##ga; \ + state[ 6] = X##ge; \ + output[ 6] = X##ge; \ + state[ 7] = X##gi; \ + output[ 7] = X##gi; \ + state[ 8] = X##go; \ + output[ 8] = X##go; \ + state[ 9] = X##gu; \ + output[ 9] = X##gu; \ + state[10] = X##ka; \ + output[10] = X##ka; \ + state[11] = X##ke; \ + output[11] = X##ke; \ + state[12] = X##ki; \ + output[12]= X##ki; \ + state[13] = X##ko; \ + output[13] = X##ko; \ + state[14] = X##ku; \ + output[14] = X##ku; \ + state[15] = X##ma; \ + output[15] = X##ma; \ + if (laneCount < 24) { \ + if (laneCount < 20) { \ + if (laneCount < 18) { \ + state[16] = X##me; \ From pypy.commits at gmail.com Wed Jan 3 17:54:46 2018 From: pypy.commits at gmail.com (amauryfa) Date: Wed, 03 Jan 2018 14:54:46 -0800 (PST) Subject: [pypy-commit] pypy py3.6: Add BLAKE2 (blake2b and blake2s) to hashlib. Message-ID: <5a4d5f36.adaedf0a.4370c.6fc2@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r93617:f6a0b040703c Date: 2017-12-22 21:03 +0100 http://bitbucket.org/pypy/pypy/changeset/f6a0b040703c/ Log: Add BLAKE2 (blake2b and blake2s) to hashlib. diff too long, truncating to 2000 out of 3659 lines diff --git a/lib-python/3/hashlib.py b/lib-python/3/hashlib.py --- a/lib-python/3/hashlib.py +++ b/lib-python/3/hashlib.py @@ -56,9 +56,9 @@ # This tuple and __get_builtin_constructor() must be modified if a new # always available algorithm is added. __always_supported = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', + 'blake2b', 'blake2s', # FIXME the following algorithms were added in # cpython3.6 but are missing in pypy - # 'blake2b', 'blake2s', # 'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', # 'shake_128', 'shake_256' ) diff --git a/lib_pypy/_blake2/__init__.py b/lib_pypy/_blake2/__init__.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_blake2/__init__.py @@ -0,0 +1,151 @@ +import codecs + +def make_blake_hash(class_name, cffi_mod): + _ffi = cffi_mod.ffi + _lib = cffi_mod.lib + + class _blake: + SALT_SIZE = _lib.BLAKE_SALTBYTES + PERSON_SIZE = _lib.BLAKE_PERSONALBYTES + MAX_KEY_SIZE = _lib.BLAKE_KEYBYTES + MAX_DIGEST_SIZE = _lib.BLAKE_OUTBYTES + + def __new__(cls, string=None, *, digest_size=MAX_DIGEST_SIZE, + key=None, salt=None, person=None, fanout=1, depth=1, + leaf_size=None, node_offset=None, node_depth=0, + inner_size=0, last_node=False): + self = super().__new__(cls) + + self._param = _ffi.new("blake_param*") + self._state = _ffi.new("blake_state*") + + # Set digest size. + if not 1 <= digest_size <= self.MAX_DIGEST_SIZE: + raise ValueError( + "digest_size must be between 1 and %s bytes" % + self.MAX_DIGEST_SIZE) + self._param.digest_length = digest_size + + # Set salt parameter. + if salt is not None: + if len(salt) > self.SALT_SIZE: + raise ValueError( + "maximum salt length is %d bytes" % + self.SALT_SIZE) + _ffi.memmove(self._param.salt, salt, len(salt)) + + # Set personalization parameter. + if person: + if len(person) > _lib.BLAKE_PERSONALBYTES: + raise ValueError("maximum person length is %d bytes" % + _lib.BLAKE_PERSONALBYTES) + _ffi.memmove(self._param.personal, person, len(person)) + + # Set tree parameters. + if not 0 <= fanout <= 255: + raise ValueError("fanout must be between 0 and 255") + self._param.fanout = fanout + + if not 1 <= depth <= 255: + raise ValueError("depth must be between 1 and 255") + self._param.depth = depth + + if leaf_size is not None: + if leaf_size > 0xFFFFFFFF: + raise OverflowError("leaf_size is too large") + # NB: Simple assignment here would be incorrect on big + # endian platforms. + _lib.store32(_ffi.addressof(self._param, 'leaf_length'), + leaf_size) + + if node_offset is not None: + if class_name == 'blake2s': + if node_offset > 0xFFFFFFFFFFFF: + # maximum 2**48 - 1 + raise OverflowError("node_offset is too large") + _lib.store48(_lib.addressof_node_offset(self._param), + node_offset) + else: + # NB: Simple assignment here would be incorrect on big + # endian platforms. + _lib.store64(_lib.addressof_node_offset(self._param), + node_offset) + + if not 0 <= node_depth <= 255: + raise ValueError("node_depth must be between 0 and 255") + self._param.node_depth = node_depth + + if not 0 <= inner_size <= _lib.BLAKE_OUTBYTES: + raise ValueError("inner_size must be between 0 and is %d" % + _lib.BLAKE_OUTBYTES) + self._param.inner_length = inner_size + + # Set key length. + if key: + if len(key) > _lib.BLAKE_KEYBYTES: + raise ValueError("maximum key length is %d bytes" % + _lib.BLAKE_KEYBYTES) + self._param.key_length = len(key) + + # Initialize hash state. + if _lib.blake_init_param(self._state, self._param) < 0: + raise RuntimeError("error initializing hash state") + + # Set last node flag (must come after initialization). + self._state.last_node = last_node + + # Process key block if any. + if key: + block = _ffi.new("uint8_t[]", _lib.BLAKE_BLOCKBYTES) + _ffi.memmove(block, key, len(key)) + _lib.blake_update(self._state, block, len(block)) + # secure_zero_memory(block, sizeof(block) + + if string: + self.update(string) + return self + + @property + def name(self): + return class_name + + @property + def block_size(self): + return _lib.BLAKE_BLOCKBYTES + + @property + def digest_size(self): + return self._param.digest_length + + def update(self, data): + data = _ffi.from_buffer(data) + _lib.blake_update(self._state, data, len(data)) + + def digest(self): + digest = _ffi.new("char[]", _lib.BLAKE_OUTBYTES) + state_copy = _ffi.new("blake_state*") + _ffi.memmove(state_copy, self._state, _ffi.sizeof("blake_state")) + _lib.blake_final(state_copy, digest, self._param.digest_length) + return _ffi.unpack(digest, self._param.digest_length) + + def hexdigest(self): + return codecs.encode(self.digest(), 'hex').decode() + + def copy(self): + copy = super().__new__(type(self)) + copy._state = _ffi.new("blake_state*") + _ffi.memmove(copy._state, self._state, _ffi.sizeof("blake_state")) + copy._param = _ffi.new("blake_param*") + _ffi.memmove(copy._param, self._param, _ffi.sizeof("blake_param")) + return copy + + _blake.__name__ = class_name + _blake.__qualname__ = class_name + return _blake + + +from . import _blake2b_cffi +blake2b = make_blake_hash('blake2b', _blake2b_cffi) + +from . import _blake2s_cffi +blake2s = make_blake_hash('blake2s', _blake2s_cffi) diff --git a/lib_pypy/_blake2/_blake2_build.py b/lib_pypy/_blake2/_blake2_build.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_blake2/_blake2_build.py @@ -0,0 +1,95 @@ +import os +import sys + +from cffi import FFI + +blake_cdef = """ +#define BLAKE_OUTBYTES ... +#define BLAKE_SALTBYTES ... +#define BLAKE_BLOCKBYTES ... +#define BLAKE_PERSONALBYTES ... +#define BLAKE_KEYBYTES ... + +typedef struct { + uint8_t digest_length; + uint8_t key_length; + uint8_t fanout; + uint8_t depth; + uint32_t leaf_length; + uint8_t node_depth; + // node_offset is a bit special + uint8_t inner_length; + uint8_t salt[...]; + uint8_t personal[...]; + ...; +} blake_param ; + +typedef struct { + uint8_t last_node; + ...; +} blake_state ; + +int blake_init_param( blake_state *S, const blake_param *P ); +int blake_update( blake_state *S, const uint8_t *in, uint64_t inlen ); +int blake_final( blake_state *S, uint8_t *out, uint8_t outlen ); + +void* addressof_node_offset(blake_param *S); + +void store32(void* dst, uint32_t w); +void store48(void* dst, uint64_t w); +void store64(void* dst, uint64_t w); +""" + +blake2b_source = """ +#include "blake2.h" +#include "blake2-impl.h" + +#define BLAKE_OUTBYTES BLAKE2B_OUTBYTES +#define BLAKE_SALTBYTES BLAKE2B_SALTBYTES +#define BLAKE_BLOCKBYTES BLAKE2B_BLOCKBYTES +#define BLAKE_PERSONALBYTES BLAKE2B_PERSONALBYTES +#define BLAKE_KEYBYTES BLAKE2B_KEYBYTES + +typedef blake2b_state blake_state; +typedef blake2b_param blake_param; + +#define blake_init_param blake2b_init_param +#define blake_update blake2b_update +#define blake_final blake2b_final + +void* addressof_node_offset(blake_param *S) { + return &(S->node_offset); +} +""" + + +_libdir = os.path.join(os.path.dirname(__file__), 'impl') + +blake2b_ffi = FFI() +blake2b_ffi.cdef(blake_cdef) +blake2b_ffi.set_source( + '_blake2b_cffi', blake2b_source, + sources=[os.path.join(_libdir, 'blake2b.c'), + ], + include_dirs=[_libdir], +) + +def _replace_b2s(src): + for b, s in (('blake2b', 'blake2s'), + ('BLAKE2B', 'BLAKE2S')): + src = src.replace(b, s) + return src + +blake2s_ffi = FFI() +blake2s_ffi.cdef(blake_cdef) +blake2s_ffi.set_source( + '_blake2s_cffi', _replace_b2s(blake2b_source), + sources=[os.path.join(_libdir, 'blake2s.c'), + ], + include_dirs=[_libdir], +) + +if __name__ == '__main__': + os.chdir(os.path.dirname(__file__)) + blake2b_ffi.compile() + blake2s_ffi.compile() diff --git a/lib_pypy/_blake2/impl/blake2-config.h b/lib_pypy/_blake2/impl/blake2-config.h new file mode 100644 --- /dev/null +++ b/lib_pypy/_blake2/impl/blake2-config.h @@ -0,0 +1,74 @@ +/* + BLAKE2 reference source code package - optimized C implementations + + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. +*/ +#pragma once +#ifndef __BLAKE2_CONFIG_H__ +#define __BLAKE2_CONFIG_H__ + +/* These don't work everywhere */ +#if defined(__SSE2__) || defined(__x86_64__) || defined(__amd64__) +#define HAVE_SSE2 +#endif + +#if defined(__SSSE3__) +#define HAVE_SSSE3 +#endif + +#if defined(__SSE4_1__) +#define HAVE_SSE41 +#endif + +#if defined(__AVX__) +#define HAVE_AVX +#endif + +#if defined(__XOP__) +#define HAVE_XOP +#endif + + +#ifdef HAVE_AVX2 +#ifndef HAVE_AVX +#define HAVE_AVX +#endif +#endif + +#ifdef HAVE_XOP +#ifndef HAVE_AVX +#define HAVE_AVX +#endif +#endif + +#ifdef HAVE_AVX +#ifndef HAVE_SSE41 +#define HAVE_SSE41 +#endif +#endif + +#ifdef HAVE_SSE41 +#ifndef HAVE_SSSE3 +#define HAVE_SSSE3 +#endif +#endif + +#ifdef HAVE_SSSE3 +#define HAVE_SSE2 +#endif + +#if !defined(HAVE_SSE2) +#error "This code requires at least SSE2." +#endif + +#endif + diff --git a/lib_pypy/_blake2/impl/blake2-impl.h b/lib_pypy/_blake2/impl/blake2-impl.h new file mode 100644 --- /dev/null +++ b/lib_pypy/_blake2/impl/blake2-impl.h @@ -0,0 +1,139 @@ +/* + BLAKE2 reference source code package - optimized C implementations + + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. +*/ +#pragma once +#ifndef __BLAKE2_IMPL_H__ +#define __BLAKE2_IMPL_H__ + +#include +#include + +BLAKE2_LOCAL_INLINE(uint32_t) load32( const void *src ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + uint32_t w; + memcpy(&w, src, sizeof w); + return w; +#else + const uint8_t *p = ( const uint8_t * )src; + uint32_t w = *p++; + w |= ( uint32_t )( *p++ ) << 8; + w |= ( uint32_t )( *p++ ) << 16; + w |= ( uint32_t )( *p++ ) << 24; + return w; +#endif +} + +BLAKE2_LOCAL_INLINE(uint64_t) load64( const void *src ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + uint64_t w; + memcpy(&w, src, sizeof w); + return w; +#else + const uint8_t *p = ( const uint8_t * )src; + uint64_t w = *p++; + w |= ( uint64_t )( *p++ ) << 8; + w |= ( uint64_t )( *p++ ) << 16; + w |= ( uint64_t )( *p++ ) << 24; + w |= ( uint64_t )( *p++ ) << 32; + w |= ( uint64_t )( *p++ ) << 40; + w |= ( uint64_t )( *p++ ) << 48; + w |= ( uint64_t )( *p++ ) << 56; + return w; +#endif +} + +BLAKE2_LOCAL_INLINE(void) store32( void *dst, uint32_t w ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + memcpy(dst, &w, sizeof w); +#else + uint8_t *p = ( uint8_t * )dst; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; +#endif +} + +BLAKE2_LOCAL_INLINE(void) store64( void *dst, uint64_t w ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + memcpy(dst, &w, sizeof w); +#else + uint8_t *p = ( uint8_t * )dst; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; +#endif +} + +BLAKE2_LOCAL_INLINE(uint64_t) load48( const void *src ) +{ + const uint8_t *p = ( const uint8_t * )src; + uint64_t w = *p++; + w |= ( uint64_t )( *p++ ) << 8; + w |= ( uint64_t )( *p++ ) << 16; + w |= ( uint64_t )( *p++ ) << 24; + w |= ( uint64_t )( *p++ ) << 32; + w |= ( uint64_t )( *p++ ) << 40; + return w; +} + +BLAKE2_LOCAL_INLINE(void) store48( void *dst, uint64_t w ) +{ + uint8_t *p = ( uint8_t * )dst; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; +} + +BLAKE2_LOCAL_INLINE(uint32_t) rotl32( const uint32_t w, const unsigned c ) +{ + return ( w << c ) | ( w >> ( 32 - c ) ); +} + +BLAKE2_LOCAL_INLINE(uint64_t) rotl64( const uint64_t w, const unsigned c ) +{ + return ( w << c ) | ( w >> ( 64 - c ) ); +} + +BLAKE2_LOCAL_INLINE(uint32_t) rotr32( const uint32_t w, const unsigned c ) +{ + return ( w >> c ) | ( w << ( 32 - c ) ); +} + +BLAKE2_LOCAL_INLINE(uint64_t) rotr64( const uint64_t w, const unsigned c ) +{ + return ( w >> c ) | ( w << ( 64 - c ) ); +} + +/* prevents compiler optimizing out memset() */ +BLAKE2_LOCAL_INLINE(void) secure_zero_memory(void *v, size_t n) +{ + static void *(*const volatile memset_v)(void *, int, size_t) = &memset; + memset_v(v, 0, n); +} + +#endif + diff --git a/lib_pypy/_blake2/impl/blake2.h b/lib_pypy/_blake2/impl/blake2.h new file mode 100644 --- /dev/null +++ b/lib_pypy/_blake2/impl/blake2.h @@ -0,0 +1,161 @@ +/* + BLAKE2 reference source code package - reference C implementations + + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. +*/ +#pragma once +#ifndef __BLAKE2_H__ +#define __BLAKE2_H__ + +#include +#include + +#ifdef BLAKE2_NO_INLINE +#define BLAKE2_LOCAL_INLINE(type) static type +#endif + +#ifndef BLAKE2_LOCAL_INLINE +#define BLAKE2_LOCAL_INLINE(type) static inline type +#endif + +#if defined(__cplusplus) +extern "C" { +#endif + + enum blake2s_constant + { + BLAKE2S_BLOCKBYTES = 64, + BLAKE2S_OUTBYTES = 32, + BLAKE2S_KEYBYTES = 32, + BLAKE2S_SALTBYTES = 8, + BLAKE2S_PERSONALBYTES = 8 + }; + + enum blake2b_constant + { + BLAKE2B_BLOCKBYTES = 128, + BLAKE2B_OUTBYTES = 64, + BLAKE2B_KEYBYTES = 64, + BLAKE2B_SALTBYTES = 16, + BLAKE2B_PERSONALBYTES = 16 + }; + + typedef struct __blake2s_state + { + uint32_t h[8]; + uint32_t t[2]; + uint32_t f[2]; + uint8_t buf[2 * BLAKE2S_BLOCKBYTES]; + size_t buflen; + uint8_t last_node; + } blake2s_state; + + typedef struct __blake2b_state + { + uint64_t h[8]; + uint64_t t[2]; + uint64_t f[2]; + uint8_t buf[2 * BLAKE2B_BLOCKBYTES]; + size_t buflen; + uint8_t last_node; + } blake2b_state; + + typedef struct __blake2sp_state + { + blake2s_state S[8][1]; + blake2s_state R[1]; + uint8_t buf[8 * BLAKE2S_BLOCKBYTES]; + size_t buflen; + } blake2sp_state; + + typedef struct __blake2bp_state + { + blake2b_state S[4][1]; + blake2b_state R[1]; + uint8_t buf[4 * BLAKE2B_BLOCKBYTES]; + size_t buflen; + } blake2bp_state; + + +#pragma pack(push, 1) + typedef struct __blake2s_param + { + uint8_t digest_length; /* 1 */ + uint8_t key_length; /* 2 */ + uint8_t fanout; /* 3 */ + uint8_t depth; /* 4 */ + uint32_t leaf_length; /* 8 */ + uint8_t node_offset[6];// 14 + uint8_t node_depth; /* 15 */ + uint8_t inner_length; /* 16 */ + /* uint8_t reserved[0]; */ + uint8_t salt[BLAKE2S_SALTBYTES]; /* 24 */ + uint8_t personal[BLAKE2S_PERSONALBYTES]; /* 32 */ + } blake2s_param; + + typedef struct __blake2b_param + { + uint8_t digest_length; /* 1 */ + uint8_t key_length; /* 2 */ + uint8_t fanout; /* 3 */ + uint8_t depth; /* 4 */ + uint32_t leaf_length; /* 8 */ + uint64_t node_offset; /* 16 */ + uint8_t node_depth; /* 17 */ + uint8_t inner_length; /* 18 */ + uint8_t reserved[14]; /* 32 */ + uint8_t salt[BLAKE2B_SALTBYTES]; /* 48 */ + uint8_t personal[BLAKE2B_PERSONALBYTES]; /* 64 */ + } blake2b_param; +#pragma pack(pop) + + /* Streaming API */ + int blake2s_init( blake2s_state *S, const uint8_t outlen ); + int blake2s_init_key( blake2s_state *S, const uint8_t outlen, const void *key, const uint8_t keylen ); + int blake2s_init_param( blake2s_state *S, const blake2s_param *P ); + int blake2s_update( blake2s_state *S, const uint8_t *in, uint64_t inlen ); + int blake2s_final( blake2s_state *S, uint8_t *out, uint8_t outlen ); + + int blake2b_init( blake2b_state *S, const uint8_t outlen ); + int blake2b_init_key( blake2b_state *S, const uint8_t outlen, const void *key, const uint8_t keylen ); + int blake2b_init_param( blake2b_state *S, const blake2b_param *P ); + int blake2b_update( blake2b_state *S, const uint8_t *in, uint64_t inlen ); + int blake2b_final( blake2b_state *S, uint8_t *out, uint8_t outlen ); + + int blake2sp_init( blake2sp_state *S, const uint8_t outlen ); + int blake2sp_init_key( blake2sp_state *S, const uint8_t outlen, const void *key, const uint8_t keylen ); + int blake2sp_update( blake2sp_state *S, const uint8_t *in, uint64_t inlen ); + int blake2sp_final( blake2sp_state *S, uint8_t *out, uint8_t outlen ); + + int blake2bp_init( blake2bp_state *S, const uint8_t outlen ); + int blake2bp_init_key( blake2bp_state *S, const uint8_t outlen, const void *key, const uint8_t keylen ); + int blake2bp_update( blake2bp_state *S, const uint8_t *in, uint64_t inlen ); + int blake2bp_final( blake2bp_state *S, uint8_t *out, uint8_t outlen ); + + /* Simple API */ + int blake2s( uint8_t *out, const void *in, const void *key, const uint8_t outlen, const uint64_t inlen, uint8_t keylen ); + int blake2b( uint8_t *out, const void *in, const void *key, const uint8_t outlen, const uint64_t inlen, uint8_t keylen ); + + int blake2sp( uint8_t *out, const void *in, const void *key, const uint8_t outlen, const uint64_t inlen, uint8_t keylen ); + int blake2bp( uint8_t *out, const void *in, const void *key, const uint8_t outlen, const uint64_t inlen, uint8_t keylen ); + + static inline int blake2( uint8_t *out, const void *in, const void *key, const uint8_t outlen, const uint64_t inlen, uint8_t keylen ) + { + return blake2b( out, in, key, outlen, inlen, keylen ); + } + +#if defined(__cplusplus) +} +#endif + +#endif + diff --git a/lib_pypy/_blake2/impl/blake2b-load-sse2.h b/lib_pypy/_blake2/impl/blake2b-load-sse2.h new file mode 100644 --- /dev/null +++ b/lib_pypy/_blake2/impl/blake2b-load-sse2.h @@ -0,0 +1,70 @@ +/* + BLAKE2 reference source code package - optimized C implementations + + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. +*/ +#pragma once +#ifndef __BLAKE2B_LOAD_SSE2_H__ +#define __BLAKE2B_LOAD_SSE2_H__ + +#define LOAD_MSG_0_1(b0, b1) b0 = _mm_set_epi64x(m2, m0); b1 = _mm_set_epi64x(m6, m4) +#define LOAD_MSG_0_2(b0, b1) b0 = _mm_set_epi64x(m3, m1); b1 = _mm_set_epi64x(m7, m5) +#define LOAD_MSG_0_3(b0, b1) b0 = _mm_set_epi64x(m10, m8); b1 = _mm_set_epi64x(m14, m12) +#define LOAD_MSG_0_4(b0, b1) b0 = _mm_set_epi64x(m11, m9); b1 = _mm_set_epi64x(m15, m13) +#define LOAD_MSG_1_1(b0, b1) b0 = _mm_set_epi64x(m4, m14); b1 = _mm_set_epi64x(m13, m9) +#define LOAD_MSG_1_2(b0, b1) b0 = _mm_set_epi64x(m8, m10); b1 = _mm_set_epi64x(m6, m15) +#define LOAD_MSG_1_3(b0, b1) b0 = _mm_set_epi64x(m0, m1); b1 = _mm_set_epi64x(m5, m11) +#define LOAD_MSG_1_4(b0, b1) b0 = _mm_set_epi64x(m2, m12); b1 = _mm_set_epi64x(m3, m7) +#define LOAD_MSG_2_1(b0, b1) b0 = _mm_set_epi64x(m12, m11); b1 = _mm_set_epi64x(m15, m5) +#define LOAD_MSG_2_2(b0, b1) b0 = _mm_set_epi64x(m0, m8); b1 = _mm_set_epi64x(m13, m2) +#define LOAD_MSG_2_3(b0, b1) b0 = _mm_set_epi64x(m3, m10); b1 = _mm_set_epi64x(m9, m7) +#define LOAD_MSG_2_4(b0, b1) b0 = _mm_set_epi64x(m6, m14); b1 = _mm_set_epi64x(m4, m1) +#define LOAD_MSG_3_1(b0, b1) b0 = _mm_set_epi64x(m3, m7); b1 = _mm_set_epi64x(m11, m13) +#define LOAD_MSG_3_2(b0, b1) b0 = _mm_set_epi64x(m1, m9); b1 = _mm_set_epi64x(m14, m12) +#define LOAD_MSG_3_3(b0, b1) b0 = _mm_set_epi64x(m5, m2); b1 = _mm_set_epi64x(m15, m4) +#define LOAD_MSG_3_4(b0, b1) b0 = _mm_set_epi64x(m10, m6); b1 = _mm_set_epi64x(m8, m0) +#define LOAD_MSG_4_1(b0, b1) b0 = _mm_set_epi64x(m5, m9); b1 = _mm_set_epi64x(m10, m2) +#define LOAD_MSG_4_2(b0, b1) b0 = _mm_set_epi64x(m7, m0); b1 = _mm_set_epi64x(m15, m4) +#define LOAD_MSG_4_3(b0, b1) b0 = _mm_set_epi64x(m11, m14); b1 = _mm_set_epi64x(m3, m6) +#define LOAD_MSG_4_4(b0, b1) b0 = _mm_set_epi64x(m12, m1); b1 = _mm_set_epi64x(m13, m8) +#define LOAD_MSG_5_1(b0, b1) b0 = _mm_set_epi64x(m6, m2); b1 = _mm_set_epi64x(m8, m0) +#define LOAD_MSG_5_2(b0, b1) b0 = _mm_set_epi64x(m10, m12); b1 = _mm_set_epi64x(m3, m11) +#define LOAD_MSG_5_3(b0, b1) b0 = _mm_set_epi64x(m7, m4); b1 = _mm_set_epi64x(m1, m15) +#define LOAD_MSG_5_4(b0, b1) b0 = _mm_set_epi64x(m5, m13); b1 = _mm_set_epi64x(m9, m14) +#define LOAD_MSG_6_1(b0, b1) b0 = _mm_set_epi64x(m1, m12); b1 = _mm_set_epi64x(m4, m14) +#define LOAD_MSG_6_2(b0, b1) b0 = _mm_set_epi64x(m15, m5); b1 = _mm_set_epi64x(m10, m13) +#define LOAD_MSG_6_3(b0, b1) b0 = _mm_set_epi64x(m6, m0); b1 = _mm_set_epi64x(m8, m9) +#define LOAD_MSG_6_4(b0, b1) b0 = _mm_set_epi64x(m3, m7); b1 = _mm_set_epi64x(m11, m2) +#define LOAD_MSG_7_1(b0, b1) b0 = _mm_set_epi64x(m7, m13); b1 = _mm_set_epi64x(m3, m12) +#define LOAD_MSG_7_2(b0, b1) b0 = _mm_set_epi64x(m14, m11); b1 = _mm_set_epi64x(m9, m1) +#define LOAD_MSG_7_3(b0, b1) b0 = _mm_set_epi64x(m15, m5); b1 = _mm_set_epi64x(m2, m8) +#define LOAD_MSG_7_4(b0, b1) b0 = _mm_set_epi64x(m4, m0); b1 = _mm_set_epi64x(m10, m6) +#define LOAD_MSG_8_1(b0, b1) b0 = _mm_set_epi64x(m14, m6); b1 = _mm_set_epi64x(m0, m11) +#define LOAD_MSG_8_2(b0, b1) b0 = _mm_set_epi64x(m9, m15); b1 = _mm_set_epi64x(m8, m3) +#define LOAD_MSG_8_3(b0, b1) b0 = _mm_set_epi64x(m13, m12); b1 = _mm_set_epi64x(m10, m1) +#define LOAD_MSG_8_4(b0, b1) b0 = _mm_set_epi64x(m7, m2); b1 = _mm_set_epi64x(m5, m4) +#define LOAD_MSG_9_1(b0, b1) b0 = _mm_set_epi64x(m8, m10); b1 = _mm_set_epi64x(m1, m7) +#define LOAD_MSG_9_2(b0, b1) b0 = _mm_set_epi64x(m4, m2); b1 = _mm_set_epi64x(m5, m6) +#define LOAD_MSG_9_3(b0, b1) b0 = _mm_set_epi64x(m9, m15); b1 = _mm_set_epi64x(m13, m3) +#define LOAD_MSG_9_4(b0, b1) b0 = _mm_set_epi64x(m14, m11); b1 = _mm_set_epi64x(m0, m12) +#define LOAD_MSG_10_1(b0, b1) b0 = _mm_set_epi64x(m2, m0); b1 = _mm_set_epi64x(m6, m4) +#define LOAD_MSG_10_2(b0, b1) b0 = _mm_set_epi64x(m3, m1); b1 = _mm_set_epi64x(m7, m5) +#define LOAD_MSG_10_3(b0, b1) b0 = _mm_set_epi64x(m10, m8); b1 = _mm_set_epi64x(m14, m12) +#define LOAD_MSG_10_4(b0, b1) b0 = _mm_set_epi64x(m11, m9); b1 = _mm_set_epi64x(m15, m13) +#define LOAD_MSG_11_1(b0, b1) b0 = _mm_set_epi64x(m4, m14); b1 = _mm_set_epi64x(m13, m9) +#define LOAD_MSG_11_2(b0, b1) b0 = _mm_set_epi64x(m8, m10); b1 = _mm_set_epi64x(m6, m15) +#define LOAD_MSG_11_3(b0, b1) b0 = _mm_set_epi64x(m0, m1); b1 = _mm_set_epi64x(m5, m11) +#define LOAD_MSG_11_4(b0, b1) b0 = _mm_set_epi64x(m2, m12); b1 = _mm_set_epi64x(m3, m7) + + +#endif + diff --git a/lib_pypy/_blake2/impl/blake2b-load-sse41.h b/lib_pypy/_blake2/impl/blake2b-load-sse41.h new file mode 100644 --- /dev/null +++ b/lib_pypy/_blake2/impl/blake2b-load-sse41.h @@ -0,0 +1,404 @@ +/* + BLAKE2 reference source code package - optimized C implementations + + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. +*/ +#pragma once +#ifndef __BLAKE2B_LOAD_SSE41_H__ +#define __BLAKE2B_LOAD_SSE41_H__ + +#define LOAD_MSG_0_1(b0, b1) \ +do \ +{ \ +b0 = _mm_unpacklo_epi64(m0, m1); \ +b1 = _mm_unpacklo_epi64(m2, m3); \ +} while(0) + + +#define LOAD_MSG_0_2(b0, b1) \ +do \ +{ \ +b0 = _mm_unpackhi_epi64(m0, m1); \ +b1 = _mm_unpackhi_epi64(m2, m3); \ +} while(0) + + +#define LOAD_MSG_0_3(b0, b1) \ +do \ +{ \ +b0 = _mm_unpacklo_epi64(m4, m5); \ +b1 = _mm_unpacklo_epi64(m6, m7); \ +} while(0) + + +#define LOAD_MSG_0_4(b0, b1) \ +do \ +{ \ +b0 = _mm_unpackhi_epi64(m4, m5); \ +b1 = _mm_unpackhi_epi64(m6, m7); \ +} while(0) + + +#define LOAD_MSG_1_1(b0, b1) \ +do \ +{ \ +b0 = _mm_unpacklo_epi64(m7, m2); \ +b1 = _mm_unpackhi_epi64(m4, m6); \ +} while(0) + + +#define LOAD_MSG_1_2(b0, b1) \ +do \ +{ \ +b0 = _mm_unpacklo_epi64(m5, m4); \ +b1 = _mm_alignr_epi8(m3, m7, 8); \ +} while(0) + + +#define LOAD_MSG_1_3(b0, b1) \ +do \ +{ \ +b0 = _mm_shuffle_epi32(m0, _MM_SHUFFLE(1,0,3,2)); \ +b1 = _mm_unpackhi_epi64(m5, m2); \ +} while(0) + + +#define LOAD_MSG_1_4(b0, b1) \ +do \ +{ \ +b0 = _mm_unpacklo_epi64(m6, m1); \ +b1 = _mm_unpackhi_epi64(m3, m1); \ +} while(0) + + +#define LOAD_MSG_2_1(b0, b1) \ +do \ +{ \ +b0 = _mm_alignr_epi8(m6, m5, 8); \ +b1 = _mm_unpackhi_epi64(m2, m7); \ +} while(0) + + +#define LOAD_MSG_2_2(b0, b1) \ +do \ +{ \ +b0 = _mm_unpacklo_epi64(m4, m0); \ +b1 = _mm_blend_epi16(m1, m6, 0xF0); \ +} while(0) + + +#define LOAD_MSG_2_3(b0, b1) \ +do \ +{ \ +b0 = _mm_blend_epi16(m5, m1, 0xF0); \ +b1 = _mm_unpackhi_epi64(m3, m4); \ +} while(0) + + +#define LOAD_MSG_2_4(b0, b1) \ +do \ +{ \ +b0 = _mm_unpacklo_epi64(m7, m3); \ +b1 = _mm_alignr_epi8(m2, m0, 8); \ +} while(0) + + +#define LOAD_MSG_3_1(b0, b1) \ +do \ +{ \ +b0 = _mm_unpackhi_epi64(m3, m1); \ +b1 = _mm_unpackhi_epi64(m6, m5); \ +} while(0) + + +#define LOAD_MSG_3_2(b0, b1) \ +do \ +{ \ +b0 = _mm_unpackhi_epi64(m4, m0); \ +b1 = _mm_unpacklo_epi64(m6, m7); \ +} while(0) + + +#define LOAD_MSG_3_3(b0, b1) \ +do \ +{ \ +b0 = _mm_blend_epi16(m1, m2, 0xF0); \ +b1 = _mm_blend_epi16(m2, m7, 0xF0); \ +} while(0) + + +#define LOAD_MSG_3_4(b0, b1) \ +do \ +{ \ +b0 = _mm_unpacklo_epi64(m3, m5); \ +b1 = _mm_unpacklo_epi64(m0, m4); \ +} while(0) + + +#define LOAD_MSG_4_1(b0, b1) \ +do \ +{ \ +b0 = _mm_unpackhi_epi64(m4, m2); \ +b1 = _mm_unpacklo_epi64(m1, m5); \ +} while(0) + + +#define LOAD_MSG_4_2(b0, b1) \ +do \ +{ \ +b0 = _mm_blend_epi16(m0, m3, 0xF0); \ +b1 = _mm_blend_epi16(m2, m7, 0xF0); \ +} while(0) + + +#define LOAD_MSG_4_3(b0, b1) \ +do \ +{ \ +b0 = _mm_blend_epi16(m7, m5, 0xF0); \ +b1 = _mm_blend_epi16(m3, m1, 0xF0); \ +} while(0) + + +#define LOAD_MSG_4_4(b0, b1) \ +do \ +{ \ +b0 = _mm_alignr_epi8(m6, m0, 8); \ +b1 = _mm_blend_epi16(m4, m6, 0xF0); \ +} while(0) + + +#define LOAD_MSG_5_1(b0, b1) \ +do \ +{ \ +b0 = _mm_unpacklo_epi64(m1, m3); \ +b1 = _mm_unpacklo_epi64(m0, m4); \ +} while(0) + + +#define LOAD_MSG_5_2(b0, b1) \ +do \ +{ \ +b0 = _mm_unpacklo_epi64(m6, m5); \ +b1 = _mm_unpackhi_epi64(m5, m1); \ +} while(0) + + +#define LOAD_MSG_5_3(b0, b1) \ +do \ +{ \ +b0 = _mm_blend_epi16(m2, m3, 0xF0); \ +b1 = _mm_unpackhi_epi64(m7, m0); \ +} while(0) + + +#define LOAD_MSG_5_4(b0, b1) \ +do \ +{ \ +b0 = _mm_unpackhi_epi64(m6, m2); \ +b1 = _mm_blend_epi16(m7, m4, 0xF0); \ +} while(0) + + +#define LOAD_MSG_6_1(b0, b1) \ +do \ +{ \ +b0 = _mm_blend_epi16(m6, m0, 0xF0); \ +b1 = _mm_unpacklo_epi64(m7, m2); \ +} while(0) + + +#define LOAD_MSG_6_2(b0, b1) \ +do \ +{ \ +b0 = _mm_unpackhi_epi64(m2, m7); \ +b1 = _mm_alignr_epi8(m5, m6, 8); \ +} while(0) + + +#define LOAD_MSG_6_3(b0, b1) \ +do \ +{ \ +b0 = _mm_unpacklo_epi64(m0, m3); \ +b1 = _mm_shuffle_epi32(m4, _MM_SHUFFLE(1,0,3,2)); \ +} while(0) + + +#define LOAD_MSG_6_4(b0, b1) \ +do \ +{ \ +b0 = _mm_unpackhi_epi64(m3, m1); \ +b1 = _mm_blend_epi16(m1, m5, 0xF0); \ +} while(0) + + +#define LOAD_MSG_7_1(b0, b1) \ +do \ +{ \ +b0 = _mm_unpackhi_epi64(m6, m3); \ +b1 = _mm_blend_epi16(m6, m1, 0xF0); \ +} while(0) + + +#define LOAD_MSG_7_2(b0, b1) \ +do \ +{ \ +b0 = _mm_alignr_epi8(m7, m5, 8); \ +b1 = _mm_unpackhi_epi64(m0, m4); \ +} while(0) + + +#define LOAD_MSG_7_3(b0, b1) \ +do \ +{ \ +b0 = _mm_unpackhi_epi64(m2, m7); \ +b1 = _mm_unpacklo_epi64(m4, m1); \ +} while(0) + + +#define LOAD_MSG_7_4(b0, b1) \ +do \ +{ \ +b0 = _mm_unpacklo_epi64(m0, m2); \ +b1 = _mm_unpacklo_epi64(m3, m5); \ +} while(0) + + +#define LOAD_MSG_8_1(b0, b1) \ +do \ +{ \ +b0 = _mm_unpacklo_epi64(m3, m7); \ +b1 = _mm_alignr_epi8(m0, m5, 8); \ +} while(0) + + +#define LOAD_MSG_8_2(b0, b1) \ +do \ +{ \ +b0 = _mm_unpackhi_epi64(m7, m4); \ +b1 = _mm_alignr_epi8(m4, m1, 8); \ +} while(0) + + +#define LOAD_MSG_8_3(b0, b1) \ +do \ +{ \ +b0 = m6; \ +b1 = _mm_alignr_epi8(m5, m0, 8); \ +} while(0) + + +#define LOAD_MSG_8_4(b0, b1) \ +do \ +{ \ +b0 = _mm_blend_epi16(m1, m3, 0xF0); \ +b1 = m2; \ +} while(0) + + +#define LOAD_MSG_9_1(b0, b1) \ +do \ +{ \ +b0 = _mm_unpacklo_epi64(m5, m4); \ +b1 = _mm_unpackhi_epi64(m3, m0); \ +} while(0) + + +#define LOAD_MSG_9_2(b0, b1) \ +do \ +{ \ +b0 = _mm_unpacklo_epi64(m1, m2); \ +b1 = _mm_blend_epi16(m3, m2, 0xF0); \ +} while(0) + + +#define LOAD_MSG_9_3(b0, b1) \ +do \ +{ \ +b0 = _mm_unpackhi_epi64(m7, m4); \ +b1 = _mm_unpackhi_epi64(m1, m6); \ +} while(0) + + +#define LOAD_MSG_9_4(b0, b1) \ +do \ +{ \ +b0 = _mm_alignr_epi8(m7, m5, 8); \ +b1 = _mm_unpacklo_epi64(m6, m0); \ +} while(0) + + +#define LOAD_MSG_10_1(b0, b1) \ +do \ +{ \ +b0 = _mm_unpacklo_epi64(m0, m1); \ +b1 = _mm_unpacklo_epi64(m2, m3); \ +} while(0) + + +#define LOAD_MSG_10_2(b0, b1) \ +do \ +{ \ +b0 = _mm_unpackhi_epi64(m0, m1); \ +b1 = _mm_unpackhi_epi64(m2, m3); \ +} while(0) + + +#define LOAD_MSG_10_3(b0, b1) \ +do \ +{ \ +b0 = _mm_unpacklo_epi64(m4, m5); \ +b1 = _mm_unpacklo_epi64(m6, m7); \ +} while(0) + + +#define LOAD_MSG_10_4(b0, b1) \ +do \ +{ \ +b0 = _mm_unpackhi_epi64(m4, m5); \ +b1 = _mm_unpackhi_epi64(m6, m7); \ +} while(0) + + +#define LOAD_MSG_11_1(b0, b1) \ +do \ +{ \ +b0 = _mm_unpacklo_epi64(m7, m2); \ +b1 = _mm_unpackhi_epi64(m4, m6); \ +} while(0) + + +#define LOAD_MSG_11_2(b0, b1) \ +do \ +{ \ +b0 = _mm_unpacklo_epi64(m5, m4); \ +b1 = _mm_alignr_epi8(m3, m7, 8); \ +} while(0) + + +#define LOAD_MSG_11_3(b0, b1) \ +do \ +{ \ +b0 = _mm_shuffle_epi32(m0, _MM_SHUFFLE(1,0,3,2)); \ +b1 = _mm_unpackhi_epi64(m5, m2); \ +} while(0) + + +#define LOAD_MSG_11_4(b0, b1) \ +do \ +{ \ +b0 = _mm_unpacklo_epi64(m6, m1); \ +b1 = _mm_unpackhi_epi64(m3, m1); \ +} while(0) + + +#endif + diff --git a/lib_pypy/_blake2/impl/blake2b-ref.c b/lib_pypy/_blake2/impl/blake2b-ref.c new file mode 100644 --- /dev/null +++ b/lib_pypy/_blake2/impl/blake2b-ref.c @@ -0,0 +1,420 @@ +/* + BLAKE2 reference source code package - reference C implementations + + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. +*/ + +#include +#include +#include + +#include "blake2.h" +#include "blake2-impl.h" + +static const uint64_t blake2b_IV[8] = +{ + 0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, + 0x3c6ef372fe94f82bULL, 0xa54ff53a5f1d36f1ULL, + 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, + 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL +}; + +static const uint8_t blake2b_sigma[12][16] = +{ + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , + { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , + { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , + { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , + { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , + { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , + { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , + { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , + { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } +}; + + +BLAKE2_LOCAL_INLINE(int) blake2b_set_lastnode( blake2b_state *S ) +{ + S->f[1] = -1; + return 0; +} + +BLAKE2_LOCAL_INLINE(int) blake2b_clear_lastnode( blake2b_state *S ) +{ + S->f[1] = 0; + return 0; +} + +/* Some helper functions, not necessarily useful */ +BLAKE2_LOCAL_INLINE(int) blake2b_is_lastblock( const blake2b_state *S ) +{ + return S->f[0] != 0; +} + +BLAKE2_LOCAL_INLINE(int) blake2b_set_lastblock( blake2b_state *S ) +{ + if( S->last_node ) blake2b_set_lastnode( S ); + + S->f[0] = -1; + return 0; +} + +BLAKE2_LOCAL_INLINE(int) blake2b_clear_lastblock( blake2b_state *S ) +{ + if( S->last_node ) blake2b_clear_lastnode( S ); + + S->f[0] = 0; + return 0; +} + +BLAKE2_LOCAL_INLINE(int) blake2b_increment_counter( blake2b_state *S, const uint64_t inc ) +{ + S->t[0] += inc; + S->t[1] += ( S->t[0] < inc ); + return 0; +} + + + +/* Parameter-related functions */ +BLAKE2_LOCAL_INLINE(int) blake2b_param_set_digest_length( blake2b_param *P, const uint8_t digest_length ) +{ + P->digest_length = digest_length; + return 0; +} + +BLAKE2_LOCAL_INLINE(int) blake2b_param_set_fanout( blake2b_param *P, const uint8_t fanout ) +{ + P->fanout = fanout; + return 0; +} + +BLAKE2_LOCAL_INLINE(int) blake2b_param_set_max_depth( blake2b_param *P, const uint8_t depth ) +{ + P->depth = depth; + return 0; +} + +BLAKE2_LOCAL_INLINE(int) blake2b_param_set_leaf_length( blake2b_param *P, const uint32_t leaf_length ) +{ + store32( &P->leaf_length, leaf_length ); + return 0; +} + +BLAKE2_LOCAL_INLINE(int) blake2b_param_set_node_offset( blake2b_param *P, const uint64_t node_offset ) +{ + store64( &P->node_offset, node_offset ); + return 0; +} + +BLAKE2_LOCAL_INLINE(int) blake2b_param_set_node_depth( blake2b_param *P, const uint8_t node_depth ) +{ + P->node_depth = node_depth; + return 0; +} + +BLAKE2_LOCAL_INLINE(int) blake2b_param_set_inner_length( blake2b_param *P, const uint8_t inner_length ) +{ + P->inner_length = inner_length; + return 0; +} + +BLAKE2_LOCAL_INLINE(int) blake2b_param_set_salt( blake2b_param *P, const uint8_t salt[BLAKE2B_SALTBYTES] ) +{ + memcpy( P->salt, salt, BLAKE2B_SALTBYTES ); + return 0; +} + +BLAKE2_LOCAL_INLINE(int) blake2b_param_set_personal( blake2b_param *P, const uint8_t personal[BLAKE2B_PERSONALBYTES] ) +{ + memcpy( P->personal, personal, BLAKE2B_PERSONALBYTES ); + return 0; +} + +BLAKE2_LOCAL_INLINE(int) blake2b_init0( blake2b_state *S ) +{ + int i; + memset( S, 0, sizeof( blake2b_state ) ); + + for( i = 0; i < 8; ++i ) S->h[i] = blake2b_IV[i]; + + return 0; +} + +/* init xors IV with input parameter block */ +int blake2b_init_param( blake2b_state *S, const blake2b_param *P ) +{ + const uint8_t *p = ( const uint8_t * )( P ); + size_t i; + + blake2b_init0( S ); + + /* IV XOR ParamBlock */ + for( i = 0; i < 8; ++i ) + S->h[i] ^= load64( p + sizeof( S->h[i] ) * i ); + + return 0; +} + + + +int blake2b_init( blake2b_state *S, const uint8_t outlen ) +{ + blake2b_param P[1]; + + if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; + + P->digest_length = outlen; + P->key_length = 0; + P->fanout = 1; + P->depth = 1; + store32( &P->leaf_length, 0 ); + store64( &P->node_offset, 0 ); + P->node_depth = 0; + P->inner_length = 0; + memset( P->reserved, 0, sizeof( P->reserved ) ); + memset( P->salt, 0, sizeof( P->salt ) ); + memset( P->personal, 0, sizeof( P->personal ) ); + return blake2b_init_param( S, P ); +} + + +int blake2b_init_key( blake2b_state *S, const uint8_t outlen, const void *key, const uint8_t keylen ) +{ + blake2b_param P[1]; + + if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; + + if ( !key || !keylen || keylen > BLAKE2B_KEYBYTES ) return -1; + + P->digest_length = outlen; + P->key_length = keylen; + P->fanout = 1; + P->depth = 1; + store32( &P->leaf_length, 0 ); + store64( &P->node_offset, 0 ); + P->node_depth = 0; + P->inner_length = 0; + memset( P->reserved, 0, sizeof( P->reserved ) ); + memset( P->salt, 0, sizeof( P->salt ) ); + memset( P->personal, 0, sizeof( P->personal ) ); + + if( blake2b_init_param( S, P ) < 0 ) return -1; + + { + uint8_t block[BLAKE2B_BLOCKBYTES]; + memset( block, 0, BLAKE2B_BLOCKBYTES ); + memcpy( block, key, keylen ); + blake2b_update( S, block, BLAKE2B_BLOCKBYTES ); + secure_zero_memory( block, BLAKE2B_BLOCKBYTES ); /* Burn the key from stack */ + } + return 0; +} + +static int blake2b_compress( blake2b_state *S, const uint8_t block[BLAKE2B_BLOCKBYTES] ) +{ + uint64_t m[16]; + uint64_t v[16]; + int i; + + for( i = 0; i < 16; ++i ) + m[i] = load64( block + i * sizeof( m[i] ) ); + + for( i = 0; i < 8; ++i ) + v[i] = S->h[i]; + + v[ 8] = blake2b_IV[0]; + v[ 9] = blake2b_IV[1]; + v[10] = blake2b_IV[2]; + v[11] = blake2b_IV[3]; + v[12] = S->t[0] ^ blake2b_IV[4]; + v[13] = S->t[1] ^ blake2b_IV[5]; + v[14] = S->f[0] ^ blake2b_IV[6]; + v[15] = S->f[1] ^ blake2b_IV[7]; +#define G(r,i,a,b,c,d) \ + do { \ + a = a + b + m[blake2b_sigma[r][2*i+0]]; \ + d = rotr64(d ^ a, 32); \ + c = c + d; \ + b = rotr64(b ^ c, 24); \ + a = a + b + m[blake2b_sigma[r][2*i+1]]; \ + d = rotr64(d ^ a, 16); \ + c = c + d; \ + b = rotr64(b ^ c, 63); \ + } while(0) +#define ROUND(r) \ + do { \ + G(r,0,v[ 0],v[ 4],v[ 8],v[12]); \ + G(r,1,v[ 1],v[ 5],v[ 9],v[13]); \ + G(r,2,v[ 2],v[ 6],v[10],v[14]); \ + G(r,3,v[ 3],v[ 7],v[11],v[15]); \ + G(r,4,v[ 0],v[ 5],v[10],v[15]); \ + G(r,5,v[ 1],v[ 6],v[11],v[12]); \ + G(r,6,v[ 2],v[ 7],v[ 8],v[13]); \ + G(r,7,v[ 3],v[ 4],v[ 9],v[14]); \ + } while(0) + ROUND( 0 ); + ROUND( 1 ); + ROUND( 2 ); + ROUND( 3 ); + ROUND( 4 ); + ROUND( 5 ); + ROUND( 6 ); + ROUND( 7 ); + ROUND( 8 ); + ROUND( 9 ); + ROUND( 10 ); + ROUND( 11 ); + + for( i = 0; i < 8; ++i ) + S->h[i] = S->h[i] ^ v[i] ^ v[i + 8]; + +#undef G +#undef ROUND + return 0; +} + +/* inlen now in bytes */ +int blake2b_update( blake2b_state *S, const uint8_t *in, uint64_t inlen ) +{ + while( inlen > 0 ) + { + size_t left = S->buflen; + size_t fill = 2 * BLAKE2B_BLOCKBYTES - left; + + if( inlen > fill ) + { + memcpy( S->buf + left, in, fill ); /* Fill buffer */ + S->buflen += fill; + blake2b_increment_counter( S, BLAKE2B_BLOCKBYTES ); + blake2b_compress( S, S->buf ); /* Compress */ + memcpy( S->buf, S->buf + BLAKE2B_BLOCKBYTES, BLAKE2B_BLOCKBYTES ); /* Shift buffer left */ + S->buflen -= BLAKE2B_BLOCKBYTES; + in += fill; + inlen -= fill; + } + else /* inlen <= fill */ + { + memcpy( S->buf + left, in, (size_t)inlen ); + S->buflen += (size_t)inlen; /* Be lazy, do not compress */ + in += inlen; + inlen -= inlen; + } + } + + return 0; +} + +/* Is this correct? */ +int blake2b_final( blake2b_state *S, uint8_t *out, uint8_t outlen ) +{ + uint8_t buffer[BLAKE2B_OUTBYTES] = {0}; + int i; + + if( out == NULL || outlen == 0 || outlen > BLAKE2B_OUTBYTES ) + return -1; + + if( blake2b_is_lastblock( S ) ) + return -1; + + if( S->buflen > BLAKE2B_BLOCKBYTES ) + { + blake2b_increment_counter( S, BLAKE2B_BLOCKBYTES ); + blake2b_compress( S, S->buf ); + S->buflen -= BLAKE2B_BLOCKBYTES; + memmove( S->buf, S->buf + BLAKE2B_BLOCKBYTES, S->buflen ); + } + + blake2b_increment_counter( S, S->buflen ); + blake2b_set_lastblock( S ); + memset( S->buf + S->buflen, 0, 2 * BLAKE2B_BLOCKBYTES - S->buflen ); /* Padding */ + blake2b_compress( S, S->buf ); + + for( i = 0; i < 8; ++i ) /* Output full hash to temp buffer */ + store64( buffer + sizeof( S->h[i] ) * i, S->h[i] ); + + memcpy( out, buffer, outlen ); + return 0; +} + +/* inlen, at least, should be uint64_t. Others can be size_t. */ +int blake2b( uint8_t *out, const void *in, const void *key, const uint8_t outlen, const uint64_t inlen, uint8_t keylen ) +{ + blake2b_state S[1]; + + /* Verify parameters */ + if ( NULL == in && inlen > 0 ) return -1; + + if ( NULL == out ) return -1; + + if( NULL == key && keylen > 0 ) return -1; + + if( !outlen || outlen > BLAKE2B_OUTBYTES ) return -1; + + if( keylen > BLAKE2B_KEYBYTES ) return -1; + + if( keylen > 0 ) + { + if( blake2b_init_key( S, outlen, key, keylen ) < 0 ) return -1; + } + else + { + if( blake2b_init( S, outlen ) < 0 ) return -1; + } + + blake2b_update( S, ( const uint8_t * )in, inlen ); + blake2b_final( S, out, outlen ); + return 0; +} + +#if defined(SUPERCOP) +int crypto_hash( unsigned char *out, unsigned char *in, unsigned long long inlen ) +{ + return blake2b( out, in, NULL, BLAKE2B_OUTBYTES, inlen, 0 ); +} +#endif + +#if defined(BLAKE2B_SELFTEST) +#include +#include "blake2-kat.h" +int main( int argc, char **argv ) +{ + uint8_t key[BLAKE2B_KEYBYTES]; + uint8_t buf[KAT_LENGTH]; + size_t i; + + for( i = 0; i < BLAKE2B_KEYBYTES; ++i ) + key[i] = ( uint8_t )i; + + for( i = 0; i < KAT_LENGTH; ++i ) + buf[i] = ( uint8_t )i; + + for( i = 0; i < KAT_LENGTH; ++i ) + { + uint8_t hash[BLAKE2B_OUTBYTES]; + blake2b( hash, buf, key, BLAKE2B_OUTBYTES, i, BLAKE2B_KEYBYTES ); + + if( 0 != memcmp( hash, blake2b_keyed_kat[i], BLAKE2B_OUTBYTES ) ) + { + puts( "error" ); + return -1; + } + } + + puts( "ok" ); + return 0; +} +#endif + diff --git a/lib_pypy/_blake2/impl/blake2b-round.h b/lib_pypy/_blake2/impl/blake2b-round.h new file mode 100644 --- /dev/null +++ b/lib_pypy/_blake2/impl/blake2b-round.h @@ -0,0 +1,159 @@ +/* + BLAKE2 reference source code package - optimized C implementations + + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. +*/ +#pragma once +#ifndef __BLAKE2B_ROUND_H__ +#define __BLAKE2B_ROUND_H__ + +#define LOADU(p) _mm_loadu_si128( (const __m128i *)(p) ) +#define STOREU(p,r) _mm_storeu_si128((__m128i *)(p), r) + +#define TOF(reg) _mm_castsi128_ps((reg)) +#define TOI(reg) _mm_castps_si128((reg)) + +#define LIKELY(x) __builtin_expect((x),1) + + +/* Microarchitecture-specific macros */ +#ifndef HAVE_XOP +#ifdef HAVE_SSSE3 +#define _mm_roti_epi64(x, c) \ + (-(c) == 32) ? _mm_shuffle_epi32((x), _MM_SHUFFLE(2,3,0,1)) \ + : (-(c) == 24) ? _mm_shuffle_epi8((x), r24) \ + : (-(c) == 16) ? _mm_shuffle_epi8((x), r16) \ + : (-(c) == 63) ? _mm_xor_si128(_mm_srli_epi64((x), -(c)), _mm_add_epi64((x), (x))) \ + : _mm_xor_si128(_mm_srli_epi64((x), -(c)), _mm_slli_epi64((x), 64-(-(c)))) +#else +#define _mm_roti_epi64(r, c) _mm_xor_si128(_mm_srli_epi64( (r), -(c) ),_mm_slli_epi64( (r), 64-(-(c)) )) +#endif +#else +/* ... */ +#endif + + + +#define G1(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1) \ + row1l = _mm_add_epi64(_mm_add_epi64(row1l, b0), row2l); \ + row1h = _mm_add_epi64(_mm_add_epi64(row1h, b1), row2h); \ + \ + row4l = _mm_xor_si128(row4l, row1l); \ + row4h = _mm_xor_si128(row4h, row1h); \ + \ + row4l = _mm_roti_epi64(row4l, -32); \ + row4h = _mm_roti_epi64(row4h, -32); \ + \ + row3l = _mm_add_epi64(row3l, row4l); \ + row3h = _mm_add_epi64(row3h, row4h); \ + \ + row2l = _mm_xor_si128(row2l, row3l); \ + row2h = _mm_xor_si128(row2h, row3h); \ + \ + row2l = _mm_roti_epi64(row2l, -24); \ + row2h = _mm_roti_epi64(row2h, -24); \ + +#define G2(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1) \ + row1l = _mm_add_epi64(_mm_add_epi64(row1l, b0), row2l); \ + row1h = _mm_add_epi64(_mm_add_epi64(row1h, b1), row2h); \ + \ + row4l = _mm_xor_si128(row4l, row1l); \ + row4h = _mm_xor_si128(row4h, row1h); \ + \ + row4l = _mm_roti_epi64(row4l, -16); \ + row4h = _mm_roti_epi64(row4h, -16); \ + \ + row3l = _mm_add_epi64(row3l, row4l); \ + row3h = _mm_add_epi64(row3h, row4h); \ + \ + row2l = _mm_xor_si128(row2l, row3l); \ + row2h = _mm_xor_si128(row2h, row3h); \ + \ + row2l = _mm_roti_epi64(row2l, -63); \ + row2h = _mm_roti_epi64(row2h, -63); \ + +#if defined(HAVE_SSSE3) +#define DIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h) \ + t0 = _mm_alignr_epi8(row2h, row2l, 8); \ + t1 = _mm_alignr_epi8(row2l, row2h, 8); \ + row2l = t0; \ + row2h = t1; \ + \ + t0 = row3l; \ + row3l = row3h; \ + row3h = t0; \ + \ + t0 = _mm_alignr_epi8(row4h, row4l, 8); \ + t1 = _mm_alignr_epi8(row4l, row4h, 8); \ + row4l = t1; \ + row4h = t0; + +#define UNDIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h) \ + t0 = _mm_alignr_epi8(row2l, row2h, 8); \ + t1 = _mm_alignr_epi8(row2h, row2l, 8); \ + row2l = t0; \ + row2h = t1; \ + \ + t0 = row3l; \ + row3l = row3h; \ + row3h = t0; \ + \ + t0 = _mm_alignr_epi8(row4l, row4h, 8); \ + t1 = _mm_alignr_epi8(row4h, row4l, 8); \ + row4l = t1; \ + row4h = t0; +#else + +#define DIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h) \ + t0 = row4l;\ + t1 = row2l;\ + row4l = row3l;\ + row3l = row3h;\ + row3h = row4l;\ + row4l = _mm_unpackhi_epi64(row4h, _mm_unpacklo_epi64(t0, t0)); \ + row4h = _mm_unpackhi_epi64(t0, _mm_unpacklo_epi64(row4h, row4h)); \ + row2l = _mm_unpackhi_epi64(row2l, _mm_unpacklo_epi64(row2h, row2h)); \ + row2h = _mm_unpackhi_epi64(row2h, _mm_unpacklo_epi64(t1, t1)) + +#define UNDIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h) \ + t0 = row3l;\ + row3l = row3h;\ + row3h = t0;\ + t0 = row2l;\ + t1 = row4l;\ + row2l = _mm_unpackhi_epi64(row2h, _mm_unpacklo_epi64(row2l, row2l)); \ + row2h = _mm_unpackhi_epi64(t0, _mm_unpacklo_epi64(row2h, row2h)); \ + row4l = _mm_unpackhi_epi64(row4l, _mm_unpacklo_epi64(row4h, row4h)); \ + row4h = _mm_unpackhi_epi64(row4h, _mm_unpacklo_epi64(t1, t1)) + +#endif + +#if defined(HAVE_SSE41) +#include "blake2b-load-sse41.h" +#else +#include "blake2b-load-sse2.h" +#endif + +#define ROUND(r) \ + LOAD_MSG_ ##r ##_1(b0, b1); \ + G1(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1); \ + LOAD_MSG_ ##r ##_2(b0, b1); \ + G2(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1); \ + DIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h); \ + LOAD_MSG_ ##r ##_3(b0, b1); \ + G1(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1); \ + LOAD_MSG_ ##r ##_4(b0, b1); \ + G2(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1); \ + UNDIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h); + +#endif + diff --git a/lib_pypy/_blake2/impl/blake2b.c b/lib_pypy/_blake2/impl/blake2b.c new file mode 100644 --- /dev/null +++ b/lib_pypy/_blake2/impl/blake2b.c @@ -0,0 +1,453 @@ +/* + BLAKE2 reference source code package - optimized C implementations + + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. +*/ + +#include +#include +#include + +#include "blake2.h" +#include "blake2-impl.h" + +#include "blake2-config.h" + +#ifdef _MSC_VER +#include /* for _mm_set_epi64x */ +#endif +#include +#if defined(HAVE_SSSE3) +#include +#endif +#if defined(HAVE_SSE41) +#include +#endif +#if defined(HAVE_AVX) +#include +#endif +#if defined(HAVE_XOP) +#include +#endif + +#include "blake2b-round.h" + +static const uint64_t blake2b_IV[8] = +{ + 0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, + 0x3c6ef372fe94f82bULL, 0xa54ff53a5f1d36f1ULL, + 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, + 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL +}; + +static const uint8_t blake2b_sigma[12][16] = +{ + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , + { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , + { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , + { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , + { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , + { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , + { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , + { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , + { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } +}; + + +/* Some helper functions, not necessarily useful */ +BLAKE2_LOCAL_INLINE(int) blake2b_set_lastnode( blake2b_state *S ) +{ + S->f[1] = -1; + return 0; +} + +BLAKE2_LOCAL_INLINE(int) blake2b_clear_lastnode( blake2b_state *S ) +{ + S->f[1] = 0; + return 0; +} + +BLAKE2_LOCAL_INLINE(int) blake2b_is_lastblock( const blake2b_state *S ) +{ + return S->f[0] != 0; +} + +BLAKE2_LOCAL_INLINE(int) blake2b_set_lastblock( blake2b_state *S ) +{ + if( S->last_node ) blake2b_set_lastnode( S ); + + S->f[0] = -1; + return 0; +} + +BLAKE2_LOCAL_INLINE(int) blake2b_clear_lastblock( blake2b_state *S ) +{ + if( S->last_node ) blake2b_clear_lastnode( S ); + + S->f[0] = 0; + return 0; +} + + +BLAKE2_LOCAL_INLINE(int) blake2b_increment_counter( blake2b_state *S, const uint64_t inc ) +{ +#if __x86_64__ + /* ADD/ADC chain */ + __uint128_t t = ( ( __uint128_t )S->t[1] << 64 ) | S->t[0]; + t += inc; + S->t[0] = ( uint64_t )( t >> 0 ); + S->t[1] = ( uint64_t )( t >> 64 ); +#else + S->t[0] += inc; + S->t[1] += ( S->t[0] < inc ); +#endif + return 0; +} + + +/* Parameter-related functions */ +BLAKE2_LOCAL_INLINE(int) blake2b_param_set_digest_length( blake2b_param *P, const uint8_t digest_length ) +{ + P->digest_length = digest_length; + return 0; +} + +BLAKE2_LOCAL_INLINE(int) blake2b_param_set_fanout( blake2b_param *P, const uint8_t fanout ) +{ + P->fanout = fanout; + return 0; +} + +BLAKE2_LOCAL_INLINE(int) blake2b_param_set_max_depth( blake2b_param *P, const uint8_t depth ) +{ + P->depth = depth; + return 0; +} + +BLAKE2_LOCAL_INLINE(int) blake2b_param_set_leaf_length( blake2b_param *P, const uint32_t leaf_length ) +{ + P->leaf_length = leaf_length; + return 0; +} + +BLAKE2_LOCAL_INLINE(int) blake2b_param_set_node_offset( blake2b_param *P, const uint64_t node_offset ) +{ + P->node_offset = node_offset; + return 0; +} + +BLAKE2_LOCAL_INLINE(int) blake2b_param_set_node_depth( blake2b_param *P, const uint8_t node_depth ) +{ + P->node_depth = node_depth; + return 0; +} + +BLAKE2_LOCAL_INLINE(int) blake2b_param_set_inner_length( blake2b_param *P, const uint8_t inner_length ) +{ + P->inner_length = inner_length; + return 0; +} + +BLAKE2_LOCAL_INLINE(int) blake2b_param_set_salt( blake2b_param *P, const uint8_t salt[BLAKE2B_SALTBYTES] ) +{ + memcpy( P->salt, salt, BLAKE2B_SALTBYTES ); + return 0; +} + +BLAKE2_LOCAL_INLINE(int) blake2b_param_set_personal( blake2b_param *P, const uint8_t personal[BLAKE2B_PERSONALBYTES] ) +{ + memcpy( P->personal, personal, BLAKE2B_PERSONALBYTES ); + return 0; +} + +BLAKE2_LOCAL_INLINE(int) blake2b_init0( blake2b_state *S ) +{ + int i; + memset( S, 0, sizeof( blake2b_state ) ); + + for( i = 0; i < 8; ++i ) S->h[i] = blake2b_IV[i]; + + return 0; +} + +/* init xors IV with input parameter block */ +int blake2b_init_param( blake2b_state *S, const blake2b_param *P ) +{ + /*blake2b_init0( S ); */ + const uint8_t * v = ( const uint8_t * )( blake2b_IV ); + const uint8_t * p = ( const uint8_t * )( P ); + uint8_t * h = ( uint8_t * )( S->h ); + int i; + /* IV XOR ParamBlock */ + memset( S, 0, sizeof( blake2b_state ) ); + + for( i = 0; i < BLAKE2B_OUTBYTES; ++i ) h[i] = v[i] ^ p[i]; + + return 0; +} + + +/* Some sort of default parameter block initialization, for sequential blake2b */ +int blake2b_init( blake2b_state *S, const uint8_t outlen ) +{ + const blake2b_param P = + { + outlen, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + {0}, + {0}, + {0} + }; + + if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; + + return blake2b_init_param( S, &P ); +} + +int blake2b_init_key( blake2b_state *S, const uint8_t outlen, const void *key, const uint8_t keylen ) +{ + const blake2b_param P = + { + outlen, + keylen, + 1, + 1, + 0, + 0, + 0, + 0, + {0}, + {0}, + {0} + }; + + if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; + + if ( ( !keylen ) || keylen > BLAKE2B_KEYBYTES ) return -1; + + if( blake2b_init_param( S, &P ) < 0 ) + return 0; + + { + uint8_t block[BLAKE2B_BLOCKBYTES]; + memset( block, 0, BLAKE2B_BLOCKBYTES ); + memcpy( block, key, keylen ); + blake2b_update( S, block, BLAKE2B_BLOCKBYTES ); + secure_zero_memory( block, BLAKE2B_BLOCKBYTES ); /* Burn the key from stack */ + } + return 0; +} + +BLAKE2_LOCAL_INLINE(int) blake2b_compress( blake2b_state *S, const uint8_t block[BLAKE2B_BLOCKBYTES] ) +{ + __m128i row1l, row1h; + __m128i row2l, row2h; + __m128i row3l, row3h; From pypy.commits at gmail.com Wed Jan 3 17:55:57 2018 From: pypy.commits at gmail.com (amauryfa) Date: Wed, 03 Jan 2018 14:55:57 -0800 (PST) Subject: [pypy-commit] pypy default: Add an option to allow underscores in integer literals. RPython part. Message-ID: <5a4d5f7d.6199df0a.e181e.a805@mx.google.com> Author: Amaury Forgeot d'Arc Branch: Changeset: r93623:0d3c3f5e2bdb Date: 2018-01-03 22:51 +0100 http://bitbucket.org/pypy/pypy/changeset/0d3c3f5e2bdb/ Log: Add an option to allow underscores in integer literals. RPython part. diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py --- a/rpython/rlib/rarithmetic.py +++ b/rpython/rlib/rarithmetic.py @@ -845,7 +845,7 @@ # String parsing support # --------------------------- -def string_to_int(s, base=10): +def string_to_int(s, base=10, allow_underscores=False): """Utility to converts a string to an integer. If base is 0, the proper base is guessed based on the leading characters of 's'. Raises ParseStringError in case of error. @@ -854,7 +854,8 @@ from rpython.rlib.rstring import ( NumberStringParser, ParseStringOverflowError, strip_spaces) s = literal = strip_spaces(s) - p = NumberStringParser(s, literal, base, 'int') + p = NumberStringParser(s, literal, base, 'int', + allow_underscores=allow_underscores) base = p.base result = 0 while True: diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -272,7 +272,7 @@ @staticmethod @jit.elidable - def fromstr(s, base=0): + def fromstr(s, base=0, allow_underscores=False): """As string_to_int(), but ignores an optional 'l' or 'L' suffix and returns an rbigint.""" from rpython.rlib.rstring import NumberStringParser, \ @@ -281,7 +281,8 @@ if (s.endswith('l') or s.endswith('L')) and base < 22: # in base 22 and above, 'L' is a valid digit! try: long('L',22) s = s[:-1] - parser = NumberStringParser(s, literal, base, 'long') + parser = NumberStringParser(s, literal, base, 'long', + allow_underscores=allow_underscores) return rbigint._from_numberstring_parser(parser) @staticmethod diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py --- a/rpython/rlib/rstring.py +++ b/rpython/rlib/rstring.py @@ -417,6 +417,9 @@ def __init__(self, msg): self.msg = msg + def __str__(self): + return self.msg + class InvalidBaseError(ParseStringError): """Signals an invalid base argument""" @@ -431,7 +434,7 @@ raise ParseStringError("invalid literal for %s() with base %d" % (self.fname, self.original_base)) - def __init__(self, s, literal, base, fname): + def __init__(self, s, literal, base, fname, allow_underscores=False): self.fname = fname sign = 1 if s.startswith('-'): @@ -441,6 +444,8 @@ s = strip_spaces(s[1:]) self.sign = sign self.original_base = base + self.allow_underscores = allow_underscores + self.last_is_underscore = False if base == 0: if s.startswith('0x') or s.startswith('0X'): @@ -480,13 +485,22 @@ digit = (digit - ord('A')) + 10 elif 'a' <= c <= 'z': digit = (digit - ord('a')) + 10 + elif c == '_' and self.allow_underscores: + if self.last_is_underscore: + self.error() + self.last_is_underscore = True + self.i += 1 + return self.next_digit() else: self.error() if digit >= self.base: self.error() self.i += 1 + self.last_is_underscore = False return digit else: + if self.last_is_underscore: + self.error() return -1 def prev_digit(self): diff --git a/rpython/rlib/test/test_rarithmetic.py b/rpython/rlib/test/test_rarithmetic.py --- a/rpython/rlib/test/test_rarithmetic.py +++ b/rpython/rlib/test/test_rarithmetic.py @@ -553,6 +553,51 @@ py.test.raises(ParseStringError, string_to_int, '+'+s, base) py.test.raises(ParseStringError, string_to_int, '-'+s, base) + def test_number_underscores(self): + VALID_UNDERSCORE_LITERALS = [ + '0_0_0', + '4_2', + '1_0000_0000', + '0b1001_0100', + '0xffff_ffff', + '0o5_7_7', + '0b_0', + '0x_f', + '0o_5', + ] + INVALID_UNDERSCORE_LITERALS = [ + # Trailing underscores: + '0_', + '42_', + '1.4j_', + '0x_', + '0b1_', + '0xf_', + '0o5_', + # Underscores in the base selector: + '0_b0', + '0_xf', + '0_o5', + # Old-style octal, still disallowed: + '09_99', + # Multiple consecutive underscores: + '4_______2', + '0b1001__0100', + '0xffff__ffff', + '0x___', + '0o5__77', + '1e1__0', + ] + for x in VALID_UNDERSCORE_LITERALS: + print x + y = string_to_int(x, base=0, allow_underscores=True) + assert y == int(x.replace('_', ''), base=0) + for x in INVALID_UNDERSCORE_LITERALS: + print x + py.test.raises(ParseStringError, string_to_int, x, base=0, + allow_underscores=True) + + class TestExplicitIntsizes: _32_max = 2147483647 From pypy.commits at gmail.com Wed Jan 3 17:54:55 2018 From: pypy.commits at gmail.com (amauryfa) Date: Wed, 03 Jan 2018 14:54:55 -0800 (PST) Subject: [pypy-commit] pypy py3.6: Add an option to allow underscores in integer literals. RPython part. Message-ID: <5a4d5f3f.d198df0a.36b5e.aea2@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r93621:23240e4a895f Date: 2018-01-03 22:51 +0100 http://bitbucket.org/pypy/pypy/changeset/23240e4a895f/ Log: Add an option to allow underscores in integer literals. RPython part. diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py --- a/rpython/rlib/rarithmetic.py +++ b/rpython/rlib/rarithmetic.py @@ -845,7 +845,7 @@ # String parsing support # --------------------------- -def string_to_int(s, base=10): +def string_to_int(s, base=10, allow_underscores=False): """Utility to converts a string to an integer. If base is 0, the proper base is guessed based on the leading characters of 's'. Raises ParseStringError in case of error. @@ -854,7 +854,8 @@ from rpython.rlib.rstring import ( NumberStringParser, ParseStringOverflowError, strip_spaces) s = literal = strip_spaces(s) - p = NumberStringParser(s, literal, base, 'int') + p = NumberStringParser(s, literal, base, 'int', + allow_underscores=allow_underscores) base = p.base result = 0 while True: diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -272,7 +272,7 @@ @staticmethod @jit.elidable - def fromstr(s, base=0): + def fromstr(s, base=0, allow_underscores=False): """As string_to_int(), but ignores an optional 'l' or 'L' suffix and returns an rbigint.""" from rpython.rlib.rstring import NumberStringParser, \ @@ -281,7 +281,8 @@ if (s.endswith('l') or s.endswith('L')) and base < 22: # in base 22 and above, 'L' is a valid digit! try: long('L',22) s = s[:-1] - parser = NumberStringParser(s, literal, base, 'long') + parser = NumberStringParser(s, literal, base, 'long', + allow_underscores=allow_underscores) return rbigint._from_numberstring_parser(parser) @staticmethod diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py --- a/rpython/rlib/rstring.py +++ b/rpython/rlib/rstring.py @@ -417,6 +417,9 @@ def __init__(self, msg): self.msg = msg + def __str__(self): + return self.msg + class InvalidBaseError(ParseStringError): """Signals an invalid base argument""" @@ -431,7 +434,7 @@ raise ParseStringError("invalid literal for %s() with base %d" % (self.fname, self.original_base)) - def __init__(self, s, literal, base, fname): + def __init__(self, s, literal, base, fname, allow_underscores=False): self.fname = fname sign = 1 if s.startswith('-'): @@ -441,6 +444,8 @@ s = strip_spaces(s[1:]) self.sign = sign self.original_base = base + self.allow_underscores = allow_underscores + self.last_is_underscore = False if base == 0: if s.startswith('0x') or s.startswith('0X'): @@ -480,13 +485,22 @@ digit = (digit - ord('A')) + 10 elif 'a' <= c <= 'z': digit = (digit - ord('a')) + 10 + elif c == '_' and self.allow_underscores: + if self.last_is_underscore: + self.error() + self.last_is_underscore = True + self.i += 1 + return self.next_digit() else: self.error() if digit >= self.base: self.error() self.i += 1 + self.last_is_underscore = False return digit else: + if self.last_is_underscore: + self.error() return -1 def prev_digit(self): diff --git a/rpython/rlib/test/test_rarithmetic.py b/rpython/rlib/test/test_rarithmetic.py --- a/rpython/rlib/test/test_rarithmetic.py +++ b/rpython/rlib/test/test_rarithmetic.py @@ -553,6 +553,51 @@ py.test.raises(ParseStringError, string_to_int, '+'+s, base) py.test.raises(ParseStringError, string_to_int, '-'+s, base) + def test_number_underscores(self): + VALID_UNDERSCORE_LITERALS = [ + '0_0_0', + '4_2', + '1_0000_0000', + '0b1001_0100', + '0xffff_ffff', + '0o5_7_7', + '0b_0', + '0x_f', + '0o_5', + ] + INVALID_UNDERSCORE_LITERALS = [ + # Trailing underscores: + '0_', + '42_', + '1.4j_', + '0x_', + '0b1_', + '0xf_', + '0o5_', + # Underscores in the base selector: + '0_b0', + '0_xf', + '0_o5', + # Old-style octal, still disallowed: + '09_99', + # Multiple consecutive underscores: + '4_______2', + '0b1001__0100', + '0xffff__ffff', + '0x___', + '0o5__77', + '1e1__0', + ] + for x in VALID_UNDERSCORE_LITERALS: + print x + y = string_to_int(x, base=0, allow_underscores=True) + assert y == int(x.replace('_', ''), base=0) + for x in INVALID_UNDERSCORE_LITERALS: + print x + py.test.raises(ParseStringError, string_to_int, x, base=0, + allow_underscores=True) + + class TestExplicitIntsizes: _32_max = 2147483647 From pypy.commits at gmail.com Wed Jan 3 17:54:51 2018 From: pypy.commits at gmail.com (amauryfa) Date: Wed, 03 Jan 2018 14:54:51 -0800 (PST) Subject: [pypy-commit] pypy py3.6: Add math.tau Message-ID: <5a4d5f3b.50b91c0a.2b051.881e@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r93619:6190ca28a930 Date: 2017-12-25 19:08 +0100 http://bitbucket.org/pypy/pypy/changeset/6190ca28a930/ Log: Add math.tau diff --git a/pypy/module/math/__init__.py b/pypy/module/math/__init__.py --- a/pypy/module/math/__init__.py +++ b/pypy/module/math/__init__.py @@ -11,6 +11,7 @@ interpleveldefs = { 'e' : 'interp_math.get(space).w_e', 'pi' : 'interp_math.get(space).w_pi', + 'tau' : 'interp_math.get(space).w_tau', 'inf' : 'interp_math.get(space).w_inf', 'nan' : 'interp_math.get(space).w_nan', 'pow' : 'interp_math.pow', diff --git a/pypy/module/math/interp_math.py b/pypy/module/math/interp_math.py --- a/pypy/module/math/interp_math.py +++ b/pypy/module/math/interp_math.py @@ -10,6 +10,7 @@ def __init__(self, space): self.w_e = space.newfloat(math.e) self.w_pi = space.newfloat(math.pi) + self.w_tau = space.newfloat(math.pi * 2.0) self.w_inf = space.newfloat(rfloat.INFINITY) self.w_nan = space.newfloat(rfloat.NAN) def get(space): diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py --- a/pypy/module/math/test/test_math.py +++ b/pypy/module/math/test/test_math.py @@ -380,3 +380,7 @@ assert math.isinf(math.inf) assert math.inf > -math.inf assert math.isnan(math.nan) + + def test_pi_tau(self): + import math + assert math.tau == math.pi * 2.0 From pypy.commits at gmail.com Thu Jan 4 03:17:27 2018 From: pypy.commits at gmail.com (amauryfa) Date: Thu, 04 Jan 2018 00:17:27 -0800 (PST) Subject: [pypy-commit] pypy py3.6: warnings.warn() now has a "source" argument. Message-ID: <5a4de317.f285df0a.d1012.aebf@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r93625:dbcd63982736 Date: 2018-01-04 09:15 +0100 http://bitbucket.org/pypy/pypy/changeset/dbcd63982736/ Log: warnings.warn() now has a "source" argument. In the future, ResourceWarnings might log the traceback where this object was allocated (when tracemalloc works on PyPy) In the mean time, this fixes the last failures in test_io. diff --git a/pypy/module/_warnings/interp_warnings.py b/pypy/module/_warnings/interp_warnings.py --- a/pypy/module/_warnings/interp_warnings.py +++ b/pypy/module/_warnings/interp_warnings.py @@ -276,12 +276,12 @@ break space.call_method(w_stderr, "write", space.newunicode(message)) -def do_warn(space, w_message, w_category, stacklevel): +def do_warn(space, w_message, w_category, stacklevel, w_source=None): context_w = setup_context(space, stacklevel) - do_warn_explicit(space, w_category, w_message, context_w) + do_warn_explicit(space, w_category, w_message, context_w, w_source=w_source) def do_warn_explicit(space, w_category, w_message, context_w, - w_sourceline=None): + w_sourceline=None, w_source=None): w_filename, lineno, w_module, w_registry = context_w # normalize module @@ -342,19 +342,32 @@ if warned: # Already warned for this module return - w_show_fxn = get_warnings_attr(space, "showwarning") - if w_show_fxn is None: + + w_show_fn = get_warnings_attr(space, "_showwarnmsg") + if w_show_fn is None: show_warning(space, w_filename, lineno, w_text, w_category, w_sourceline) - else: - space.call_function( - w_show_fxn, w_message, w_category, w_filename, w_lineno) + return + + if not space.is_true(space.callable(w_show_fn)): + raise oefmt(space.w_TypeError, + "warnings._showwarnmsg() must be set to a callable") + w_message_cls = get_warnings_attr(space, "WarningMessage") + if w_message_cls is None: + raise oefmt(space.w_RuntimeError, + "unable to get warnings.WarningMessage") + w_source = w_source or space.w_None + w_msg = space.call_function( + w_message_cls, w_message, w_category, + w_filename, w_lineno, space.w_None, space.w_None, w_source) + space.call_function(w_show_fn, w_msg) + @unwrap_spec(stacklevel=int) -def warn(space, w_message, w_category=None, stacklevel=1): +def warn(space, w_message, w_category=None, stacklevel=1, w_source=None): "Issue a warning, or maybe ignore it or raise an exception." w_category = get_category(space, w_message, w_category); - do_warn(space, w_message, w_category, stacklevel) + do_warn(space, w_message, w_category, stacklevel, w_source) def get_source_line(space, w_globals, lineno): @@ -394,14 +407,15 @@ w_registry = WrappedDefault(None), w_module_globals = WrappedDefault(None)) def warn_explicit(space, w_message, w_category, w_filename, lineno, - w_module=None, w_registry=None, w_module_globals=None): + w_module=None, w_registry=None, w_module_globals=None, + w_source=None): "Low-level inferface to warnings functionality." w_source_line = get_source_line(space, w_module_globals, lineno) do_warn_explicit(space, w_category, w_message, (w_filename, lineno, w_module, w_registry), - w_source_line) + w_source_line, w_source) def filters_mutated(space): space.fromcache(State).filters_mutated(space) diff --git a/pypy/module/_warnings/test/test_warnings.py b/pypy/module/_warnings/test/test_warnings.py --- a/pypy/module/_warnings/test/test_warnings.py +++ b/pypy/module/_warnings/test/test_warnings.py @@ -42,6 +42,13 @@ _warnings.warn_explicit("some message", Warning, "", 1, module_globals=globals()) + def test_with_source(self): + import warnings, _warnings + source = [] + with warnings.catch_warnings(record=True) as w: + _warnings.warn("some message", Warning, source=source) + assert w[0].source is source + def test_default_action(self): import warnings, _warnings warnings.defaultaction = 'ignore' From pypy.commits at gmail.com Thu Jan 4 03:17:25 2018 From: pypy.commits at gmail.com (amauryfa) Date: Thu, 04 Jan 2018 00:17:25 -0800 (PST) Subject: [pypy-commit] pypy py3.6: Match CPython error message Message-ID: <5a4de315.ccc4df0a.f647d.0d52@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r93624:cd17bd421135 Date: 2018-01-04 00:06 +0100 http://bitbucket.org/pypy/pypy/changeset/cd17bd421135/ Log: Match CPython error message diff --git a/pypy/module/_io/interp_io.py b/pypy/module/_io/interp_io.py --- a/pypy/module/_io/interp_io.py +++ b/pypy/module/_io/interp_io.py @@ -78,9 +78,9 @@ if text and binary: raise oefmt(space.w_ValueError, "can't have text and binary mode at once") - if reading + writing + creating + appending > 1: + if creating + reading + writing + appending > 1: raise oefmt(space.w_ValueError, - "must have exactly one of read/write/create/append mode") + "must have exactly one of create/read/write/append mode") if binary and encoding is not None: raise oefmt(space.w_ValueError, "binary mode doesn't take an encoding argument") From pypy.commits at gmail.com Thu Jan 4 18:25:24 2018 From: pypy.commits at gmail.com (amauryfa) Date: Thu, 04 Jan 2018 15:25:24 -0800 (PST) Subject: [pypy-commit] pypy py3.6: Add Unicode database 9.0.0 Message-ID: <5a4eb7e4.79b8df0a.3c464.d200@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r93626:4d728845e198 Date: 2018-01-04 17:44 +0100 http://bitbucket.org/pypy/pypy/changeset/4d728845e198/ Log: Add Unicode database 9.0.0 diff too long, truncating to 2000 out of 75520 lines diff --git a/rpython/rlib/unicodedata/CaseFolding-9.0.0.txt b/rpython/rlib/unicodedata/CaseFolding-9.0.0.txt new file mode 100644 --- /dev/null +++ b/rpython/rlib/unicodedata/CaseFolding-9.0.0.txt @@ -0,0 +1,1495 @@ +# CaseFolding-9.0.0.txt +# Date: 2016-03-02, 18:54:54 GMT +# © 2016 Unicode®, Inc. +# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. +# For terms of use, see http://www.unicode.org/terms_of_use.html +# +# Unicode Character Database +# For documentation, see http://www.unicode.org/reports/tr44/ +# +# Case Folding Properties +# +# This file is a supplement to the UnicodeData file. +# It provides a case folding mapping generated from the Unicode Character Database. +# If all characters are mapped according to the full mapping below, then +# case differences (according to UnicodeData.txt and SpecialCasing.txt) +# are eliminated. +# +# The data supports both implementations that require simple case foldings +# (where string lengths don't change), and implementations that allow full case folding +# (where string lengths may grow). Note that where they can be supported, the +# full case foldings are superior: for example, they allow "MASSE" and "Maße" to match. +# +# All code points not listed in this file map to themselves. +# +# NOTE: case folding does not preserve normalization formats! +# +# For information on case folding, including how to have case folding +# preserve normalization formats, see Section 3.13 Default Case Algorithms in +# The Unicode Standard. +# +# ================================================================================ +# Format +# ================================================================================ +# The entries in this file are in the following machine-readable format: +# +# ; ; ; # +# +# The status field is: +# C: common case folding, common mappings shared by both simple and full mappings. +# F: full case folding, mappings that cause strings to grow in length. Multiple characters are separated by spaces. +# S: simple case folding, mappings to single characters where different from F. +# T: special case for uppercase I and dotted uppercase I +# - For non-Turkic languages, this mapping is normally not used. +# - For Turkic languages (tr, az), this mapping can be used instead of the normal mapping for these characters. +# Note that the Turkic mappings do not maintain canonical equivalence without additional processing. +# See the discussions of case mapping in the Unicode Standard for more information. +# +# Usage: +# A. To do a simple case folding, use the mappings with status C + S. +# B. To do a full case folding, use the mappings with status C + F. +# +# The mappings with status T can be used or omitted depending on the desired case-folding +# behavior. (The default option is to exclude them.) +# +# ================================================================= + +# Property: Case_Folding + +# All code points not explicitly listed for Case_Folding +# have the value C for the status field, and the code point itself for the mapping field. + +# ================================================================= +0041; C; 0061; # LATIN CAPITAL LETTER A +0042; C; 0062; # LATIN CAPITAL LETTER B +0043; C; 0063; # LATIN CAPITAL LETTER C +0044; C; 0064; # LATIN CAPITAL LETTER D +0045; C; 0065; # LATIN CAPITAL LETTER E +0046; C; 0066; # LATIN CAPITAL LETTER F +0047; C; 0067; # LATIN CAPITAL LETTER G +0048; C; 0068; # LATIN CAPITAL LETTER H +0049; C; 0069; # LATIN CAPITAL LETTER I +0049; T; 0131; # LATIN CAPITAL LETTER I +004A; C; 006A; # LATIN CAPITAL LETTER J +004B; C; 006B; # LATIN CAPITAL LETTER K +004C; C; 006C; # LATIN CAPITAL LETTER L +004D; C; 006D; # LATIN CAPITAL LETTER M +004E; C; 006E; # LATIN CAPITAL LETTER N +004F; C; 006F; # LATIN CAPITAL LETTER O +0050; C; 0070; # LATIN CAPITAL LETTER P +0051; C; 0071; # LATIN CAPITAL LETTER Q +0052; C; 0072; # LATIN CAPITAL LETTER R +0053; C; 0073; # LATIN CAPITAL LETTER S +0054; C; 0074; # LATIN CAPITAL LETTER T +0055; C; 0075; # LATIN CAPITAL LETTER U +0056; C; 0076; # LATIN CAPITAL LETTER V +0057; C; 0077; # LATIN CAPITAL LETTER W +0058; C; 0078; # LATIN CAPITAL LETTER X +0059; C; 0079; # LATIN CAPITAL LETTER Y +005A; C; 007A; # LATIN CAPITAL LETTER Z +00B5; C; 03BC; # MICRO SIGN +00C0; C; 00E0; # LATIN CAPITAL LETTER A WITH GRAVE +00C1; C; 00E1; # LATIN CAPITAL LETTER A WITH ACUTE +00C2; C; 00E2; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX +00C3; C; 00E3; # LATIN CAPITAL LETTER A WITH TILDE +00C4; C; 00E4; # LATIN CAPITAL LETTER A WITH DIAERESIS +00C5; C; 00E5; # LATIN CAPITAL LETTER A WITH RING ABOVE +00C6; C; 00E6; # LATIN CAPITAL LETTER AE +00C7; C; 00E7; # LATIN CAPITAL LETTER C WITH CEDILLA +00C8; C; 00E8; # LATIN CAPITAL LETTER E WITH GRAVE +00C9; C; 00E9; # LATIN CAPITAL LETTER E WITH ACUTE +00CA; C; 00EA; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX +00CB; C; 00EB; # LATIN CAPITAL LETTER E WITH DIAERESIS +00CC; C; 00EC; # LATIN CAPITAL LETTER I WITH GRAVE +00CD; C; 00ED; # LATIN CAPITAL LETTER I WITH ACUTE +00CE; C; 00EE; # LATIN CAPITAL LETTER I WITH CIRCUMFLEX +00CF; C; 00EF; # LATIN CAPITAL LETTER I WITH DIAERESIS +00D0; C; 00F0; # LATIN CAPITAL LETTER ETH +00D1; C; 00F1; # LATIN CAPITAL LETTER N WITH TILDE +00D2; C; 00F2; # LATIN CAPITAL LETTER O WITH GRAVE +00D3; C; 00F3; # LATIN CAPITAL LETTER O WITH ACUTE +00D4; C; 00F4; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX +00D5; C; 00F5; # LATIN CAPITAL LETTER O WITH TILDE +00D6; C; 00F6; # LATIN CAPITAL LETTER O WITH DIAERESIS +00D8; C; 00F8; # LATIN CAPITAL LETTER O WITH STROKE +00D9; C; 00F9; # LATIN CAPITAL LETTER U WITH GRAVE +00DA; C; 00FA; # LATIN CAPITAL LETTER U WITH ACUTE +00DB; C; 00FB; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX +00DC; C; 00FC; # LATIN CAPITAL LETTER U WITH DIAERESIS +00DD; C; 00FD; # LATIN CAPITAL LETTER Y WITH ACUTE +00DE; C; 00FE; # LATIN CAPITAL LETTER THORN +00DF; F; 0073 0073; # LATIN SMALL LETTER SHARP S +0100; C; 0101; # LATIN CAPITAL LETTER A WITH MACRON +0102; C; 0103; # LATIN CAPITAL LETTER A WITH BREVE +0104; C; 0105; # LATIN CAPITAL LETTER A WITH OGONEK +0106; C; 0107; # LATIN CAPITAL LETTER C WITH ACUTE +0108; C; 0109; # LATIN CAPITAL LETTER C WITH CIRCUMFLEX +010A; C; 010B; # LATIN CAPITAL LETTER C WITH DOT ABOVE +010C; C; 010D; # LATIN CAPITAL LETTER C WITH CARON +010E; C; 010F; # LATIN CAPITAL LETTER D WITH CARON +0110; C; 0111; # LATIN CAPITAL LETTER D WITH STROKE +0112; C; 0113; # LATIN CAPITAL LETTER E WITH MACRON +0114; C; 0115; # LATIN CAPITAL LETTER E WITH BREVE +0116; C; 0117; # LATIN CAPITAL LETTER E WITH DOT ABOVE +0118; C; 0119; # LATIN CAPITAL LETTER E WITH OGONEK +011A; C; 011B; # LATIN CAPITAL LETTER E WITH CARON +011C; C; 011D; # LATIN CAPITAL LETTER G WITH CIRCUMFLEX +011E; C; 011F; # LATIN CAPITAL LETTER G WITH BREVE +0120; C; 0121; # LATIN CAPITAL LETTER G WITH DOT ABOVE +0122; C; 0123; # LATIN CAPITAL LETTER G WITH CEDILLA +0124; C; 0125; # LATIN CAPITAL LETTER H WITH CIRCUMFLEX +0126; C; 0127; # LATIN CAPITAL LETTER H WITH STROKE +0128; C; 0129; # LATIN CAPITAL LETTER I WITH TILDE +012A; C; 012B; # LATIN CAPITAL LETTER I WITH MACRON +012C; C; 012D; # LATIN CAPITAL LETTER I WITH BREVE +012E; C; 012F; # LATIN CAPITAL LETTER I WITH OGONEK +0130; F; 0069 0307; # LATIN CAPITAL LETTER I WITH DOT ABOVE +0130; T; 0069; # LATIN CAPITAL LETTER I WITH DOT ABOVE +0132; C; 0133; # LATIN CAPITAL LIGATURE IJ +0134; C; 0135; # LATIN CAPITAL LETTER J WITH CIRCUMFLEX +0136; C; 0137; # LATIN CAPITAL LETTER K WITH CEDILLA +0139; C; 013A; # LATIN CAPITAL LETTER L WITH ACUTE +013B; C; 013C; # LATIN CAPITAL LETTER L WITH CEDILLA +013D; C; 013E; # LATIN CAPITAL LETTER L WITH CARON +013F; C; 0140; # LATIN CAPITAL LETTER L WITH MIDDLE DOT +0141; C; 0142; # LATIN CAPITAL LETTER L WITH STROKE +0143; C; 0144; # LATIN CAPITAL LETTER N WITH ACUTE +0145; C; 0146; # LATIN CAPITAL LETTER N WITH CEDILLA +0147; C; 0148; # LATIN CAPITAL LETTER N WITH CARON +0149; F; 02BC 006E; # LATIN SMALL LETTER N PRECEDED BY APOSTROPHE +014A; C; 014B; # LATIN CAPITAL LETTER ENG +014C; C; 014D; # LATIN CAPITAL LETTER O WITH MACRON +014E; C; 014F; # LATIN CAPITAL LETTER O WITH BREVE +0150; C; 0151; # LATIN CAPITAL LETTER O WITH DOUBLE ACUTE +0152; C; 0153; # LATIN CAPITAL LIGATURE OE +0154; C; 0155; # LATIN CAPITAL LETTER R WITH ACUTE +0156; C; 0157; # LATIN CAPITAL LETTER R WITH CEDILLA +0158; C; 0159; # LATIN CAPITAL LETTER R WITH CARON +015A; C; 015B; # LATIN CAPITAL LETTER S WITH ACUTE +015C; C; 015D; # LATIN CAPITAL LETTER S WITH CIRCUMFLEX +015E; C; 015F; # LATIN CAPITAL LETTER S WITH CEDILLA +0160; C; 0161; # LATIN CAPITAL LETTER S WITH CARON +0162; C; 0163; # LATIN CAPITAL LETTER T WITH CEDILLA +0164; C; 0165; # LATIN CAPITAL LETTER T WITH CARON +0166; C; 0167; # LATIN CAPITAL LETTER T WITH STROKE +0168; C; 0169; # LATIN CAPITAL LETTER U WITH TILDE +016A; C; 016B; # LATIN CAPITAL LETTER U WITH MACRON +016C; C; 016D; # LATIN CAPITAL LETTER U WITH BREVE +016E; C; 016F; # LATIN CAPITAL LETTER U WITH RING ABOVE +0170; C; 0171; # LATIN CAPITAL LETTER U WITH DOUBLE ACUTE +0172; C; 0173; # LATIN CAPITAL LETTER U WITH OGONEK +0174; C; 0175; # LATIN CAPITAL LETTER W WITH CIRCUMFLEX +0176; C; 0177; # LATIN CAPITAL LETTER Y WITH CIRCUMFLEX +0178; C; 00FF; # LATIN CAPITAL LETTER Y WITH DIAERESIS +0179; C; 017A; # LATIN CAPITAL LETTER Z WITH ACUTE +017B; C; 017C; # LATIN CAPITAL LETTER Z WITH DOT ABOVE +017D; C; 017E; # LATIN CAPITAL LETTER Z WITH CARON +017F; C; 0073; # LATIN SMALL LETTER LONG S +0181; C; 0253; # LATIN CAPITAL LETTER B WITH HOOK +0182; C; 0183; # LATIN CAPITAL LETTER B WITH TOPBAR +0184; C; 0185; # LATIN CAPITAL LETTER TONE SIX +0186; C; 0254; # LATIN CAPITAL LETTER OPEN O +0187; C; 0188; # LATIN CAPITAL LETTER C WITH HOOK +0189; C; 0256; # LATIN CAPITAL LETTER AFRICAN D +018A; C; 0257; # LATIN CAPITAL LETTER D WITH HOOK +018B; C; 018C; # LATIN CAPITAL LETTER D WITH TOPBAR +018E; C; 01DD; # LATIN CAPITAL LETTER REVERSED E +018F; C; 0259; # LATIN CAPITAL LETTER SCHWA +0190; C; 025B; # LATIN CAPITAL LETTER OPEN E +0191; C; 0192; # LATIN CAPITAL LETTER F WITH HOOK +0193; C; 0260; # LATIN CAPITAL LETTER G WITH HOOK +0194; C; 0263; # LATIN CAPITAL LETTER GAMMA +0196; C; 0269; # LATIN CAPITAL LETTER IOTA +0197; C; 0268; # LATIN CAPITAL LETTER I WITH STROKE +0198; C; 0199; # LATIN CAPITAL LETTER K WITH HOOK +019C; C; 026F; # LATIN CAPITAL LETTER TURNED M +019D; C; 0272; # LATIN CAPITAL LETTER N WITH LEFT HOOK +019F; C; 0275; # LATIN CAPITAL LETTER O WITH MIDDLE TILDE +01A0; C; 01A1; # LATIN CAPITAL LETTER O WITH HORN +01A2; C; 01A3; # LATIN CAPITAL LETTER OI +01A4; C; 01A5; # LATIN CAPITAL LETTER P WITH HOOK +01A6; C; 0280; # LATIN LETTER YR +01A7; C; 01A8; # LATIN CAPITAL LETTER TONE TWO +01A9; C; 0283; # LATIN CAPITAL LETTER ESH +01AC; C; 01AD; # LATIN CAPITAL LETTER T WITH HOOK +01AE; C; 0288; # LATIN CAPITAL LETTER T WITH RETROFLEX HOOK +01AF; C; 01B0; # LATIN CAPITAL LETTER U WITH HORN +01B1; C; 028A; # LATIN CAPITAL LETTER UPSILON +01B2; C; 028B; # LATIN CAPITAL LETTER V WITH HOOK +01B3; C; 01B4; # LATIN CAPITAL LETTER Y WITH HOOK +01B5; C; 01B6; # LATIN CAPITAL LETTER Z WITH STROKE +01B7; C; 0292; # LATIN CAPITAL LETTER EZH +01B8; C; 01B9; # LATIN CAPITAL LETTER EZH REVERSED +01BC; C; 01BD; # LATIN CAPITAL LETTER TONE FIVE +01C4; C; 01C6; # LATIN CAPITAL LETTER DZ WITH CARON +01C5; C; 01C6; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON +01C7; C; 01C9; # LATIN CAPITAL LETTER LJ +01C8; C; 01C9; # LATIN CAPITAL LETTER L WITH SMALL LETTER J +01CA; C; 01CC; # LATIN CAPITAL LETTER NJ +01CB; C; 01CC; # LATIN CAPITAL LETTER N WITH SMALL LETTER J +01CD; C; 01CE; # LATIN CAPITAL LETTER A WITH CARON +01CF; C; 01D0; # LATIN CAPITAL LETTER I WITH CARON +01D1; C; 01D2; # LATIN CAPITAL LETTER O WITH CARON +01D3; C; 01D4; # LATIN CAPITAL LETTER U WITH CARON +01D5; C; 01D6; # LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON +01D7; C; 01D8; # LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE +01D9; C; 01DA; # LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON +01DB; C; 01DC; # LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE +01DE; C; 01DF; # LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON +01E0; C; 01E1; # LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON +01E2; C; 01E3; # LATIN CAPITAL LETTER AE WITH MACRON +01E4; C; 01E5; # LATIN CAPITAL LETTER G WITH STROKE +01E6; C; 01E7; # LATIN CAPITAL LETTER G WITH CARON +01E8; C; 01E9; # LATIN CAPITAL LETTER K WITH CARON +01EA; C; 01EB; # LATIN CAPITAL LETTER O WITH OGONEK +01EC; C; 01ED; # LATIN CAPITAL LETTER O WITH OGONEK AND MACRON +01EE; C; 01EF; # LATIN CAPITAL LETTER EZH WITH CARON +01F0; F; 006A 030C; # LATIN SMALL LETTER J WITH CARON +01F1; C; 01F3; # LATIN CAPITAL LETTER DZ +01F2; C; 01F3; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z +01F4; C; 01F5; # LATIN CAPITAL LETTER G WITH ACUTE +01F6; C; 0195; # LATIN CAPITAL LETTER HWAIR +01F7; C; 01BF; # LATIN CAPITAL LETTER WYNN +01F8; C; 01F9; # LATIN CAPITAL LETTER N WITH GRAVE +01FA; C; 01FB; # LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE +01FC; C; 01FD; # LATIN CAPITAL LETTER AE WITH ACUTE +01FE; C; 01FF; # LATIN CAPITAL LETTER O WITH STROKE AND ACUTE +0200; C; 0201; # LATIN CAPITAL LETTER A WITH DOUBLE GRAVE +0202; C; 0203; # LATIN CAPITAL LETTER A WITH INVERTED BREVE +0204; C; 0205; # LATIN CAPITAL LETTER E WITH DOUBLE GRAVE +0206; C; 0207; # LATIN CAPITAL LETTER E WITH INVERTED BREVE +0208; C; 0209; # LATIN CAPITAL LETTER I WITH DOUBLE GRAVE +020A; C; 020B; # LATIN CAPITAL LETTER I WITH INVERTED BREVE +020C; C; 020D; # LATIN CAPITAL LETTER O WITH DOUBLE GRAVE +020E; C; 020F; # LATIN CAPITAL LETTER O WITH INVERTED BREVE +0210; C; 0211; # LATIN CAPITAL LETTER R WITH DOUBLE GRAVE +0212; C; 0213; # LATIN CAPITAL LETTER R WITH INVERTED BREVE +0214; C; 0215; # LATIN CAPITAL LETTER U WITH DOUBLE GRAVE +0216; C; 0217; # LATIN CAPITAL LETTER U WITH INVERTED BREVE +0218; C; 0219; # LATIN CAPITAL LETTER S WITH COMMA BELOW +021A; C; 021B; # LATIN CAPITAL LETTER T WITH COMMA BELOW +021C; C; 021D; # LATIN CAPITAL LETTER YOGH +021E; C; 021F; # LATIN CAPITAL LETTER H WITH CARON +0220; C; 019E; # LATIN CAPITAL LETTER N WITH LONG RIGHT LEG +0222; C; 0223; # LATIN CAPITAL LETTER OU +0224; C; 0225; # LATIN CAPITAL LETTER Z WITH HOOK +0226; C; 0227; # LATIN CAPITAL LETTER A WITH DOT ABOVE +0228; C; 0229; # LATIN CAPITAL LETTER E WITH CEDILLA +022A; C; 022B; # LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON +022C; C; 022D; # LATIN CAPITAL LETTER O WITH TILDE AND MACRON +022E; C; 022F; # LATIN CAPITAL LETTER O WITH DOT ABOVE +0230; C; 0231; # LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON +0232; C; 0233; # LATIN CAPITAL LETTER Y WITH MACRON +023A; C; 2C65; # LATIN CAPITAL LETTER A WITH STROKE +023B; C; 023C; # LATIN CAPITAL LETTER C WITH STROKE +023D; C; 019A; # LATIN CAPITAL LETTER L WITH BAR +023E; C; 2C66; # LATIN CAPITAL LETTER T WITH DIAGONAL STROKE +0241; C; 0242; # LATIN CAPITAL LETTER GLOTTAL STOP +0243; C; 0180; # LATIN CAPITAL LETTER B WITH STROKE +0244; C; 0289; # LATIN CAPITAL LETTER U BAR +0245; C; 028C; # LATIN CAPITAL LETTER TURNED V +0246; C; 0247; # LATIN CAPITAL LETTER E WITH STROKE +0248; C; 0249; # LATIN CAPITAL LETTER J WITH STROKE +024A; C; 024B; # LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL +024C; C; 024D; # LATIN CAPITAL LETTER R WITH STROKE +024E; C; 024F; # LATIN CAPITAL LETTER Y WITH STROKE +0345; C; 03B9; # COMBINING GREEK YPOGEGRAMMENI +0370; C; 0371; # GREEK CAPITAL LETTER HETA +0372; C; 0373; # GREEK CAPITAL LETTER ARCHAIC SAMPI +0376; C; 0377; # GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA +037F; C; 03F3; # GREEK CAPITAL LETTER YOT +0386; C; 03AC; # GREEK CAPITAL LETTER ALPHA WITH TONOS +0388; C; 03AD; # GREEK CAPITAL LETTER EPSILON WITH TONOS +0389; C; 03AE; # GREEK CAPITAL LETTER ETA WITH TONOS +038A; C; 03AF; # GREEK CAPITAL LETTER IOTA WITH TONOS +038C; C; 03CC; # GREEK CAPITAL LETTER OMICRON WITH TONOS +038E; C; 03CD; # GREEK CAPITAL LETTER UPSILON WITH TONOS +038F; C; 03CE; # GREEK CAPITAL LETTER OMEGA WITH TONOS +0390; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS +0391; C; 03B1; # GREEK CAPITAL LETTER ALPHA +0392; C; 03B2; # GREEK CAPITAL LETTER BETA +0393; C; 03B3; # GREEK CAPITAL LETTER GAMMA +0394; C; 03B4; # GREEK CAPITAL LETTER DELTA +0395; C; 03B5; # GREEK CAPITAL LETTER EPSILON +0396; C; 03B6; # GREEK CAPITAL LETTER ZETA +0397; C; 03B7; # GREEK CAPITAL LETTER ETA +0398; C; 03B8; # GREEK CAPITAL LETTER THETA +0399; C; 03B9; # GREEK CAPITAL LETTER IOTA +039A; C; 03BA; # GREEK CAPITAL LETTER KAPPA +039B; C; 03BB; # GREEK CAPITAL LETTER LAMDA +039C; C; 03BC; # GREEK CAPITAL LETTER MU +039D; C; 03BD; # GREEK CAPITAL LETTER NU +039E; C; 03BE; # GREEK CAPITAL LETTER XI +039F; C; 03BF; # GREEK CAPITAL LETTER OMICRON +03A0; C; 03C0; # GREEK CAPITAL LETTER PI +03A1; C; 03C1; # GREEK CAPITAL LETTER RHO +03A3; C; 03C3; # GREEK CAPITAL LETTER SIGMA +03A4; C; 03C4; # GREEK CAPITAL LETTER TAU +03A5; C; 03C5; # GREEK CAPITAL LETTER UPSILON +03A6; C; 03C6; # GREEK CAPITAL LETTER PHI +03A7; C; 03C7; # GREEK CAPITAL LETTER CHI +03A8; C; 03C8; # GREEK CAPITAL LETTER PSI +03A9; C; 03C9; # GREEK CAPITAL LETTER OMEGA +03AA; C; 03CA; # GREEK CAPITAL LETTER IOTA WITH DIALYTIKA +03AB; C; 03CB; # GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA +03B0; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS +03C2; C; 03C3; # GREEK SMALL LETTER FINAL SIGMA +03CF; C; 03D7; # GREEK CAPITAL KAI SYMBOL +03D0; C; 03B2; # GREEK BETA SYMBOL +03D1; C; 03B8; # GREEK THETA SYMBOL +03D5; C; 03C6; # GREEK PHI SYMBOL +03D6; C; 03C0; # GREEK PI SYMBOL +03D8; C; 03D9; # GREEK LETTER ARCHAIC KOPPA +03DA; C; 03DB; # GREEK LETTER STIGMA +03DC; C; 03DD; # GREEK LETTER DIGAMMA +03DE; C; 03DF; # GREEK LETTER KOPPA +03E0; C; 03E1; # GREEK LETTER SAMPI +03E2; C; 03E3; # COPTIC CAPITAL LETTER SHEI +03E4; C; 03E5; # COPTIC CAPITAL LETTER FEI +03E6; C; 03E7; # COPTIC CAPITAL LETTER KHEI +03E8; C; 03E9; # COPTIC CAPITAL LETTER HORI +03EA; C; 03EB; # COPTIC CAPITAL LETTER GANGIA +03EC; C; 03ED; # COPTIC CAPITAL LETTER SHIMA +03EE; C; 03EF; # COPTIC CAPITAL LETTER DEI +03F0; C; 03BA; # GREEK KAPPA SYMBOL +03F1; C; 03C1; # GREEK RHO SYMBOL +03F4; C; 03B8; # GREEK CAPITAL THETA SYMBOL +03F5; C; 03B5; # GREEK LUNATE EPSILON SYMBOL +03F7; C; 03F8; # GREEK CAPITAL LETTER SHO +03F9; C; 03F2; # GREEK CAPITAL LUNATE SIGMA SYMBOL +03FA; C; 03FB; # GREEK CAPITAL LETTER SAN +03FD; C; 037B; # GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL +03FE; C; 037C; # GREEK CAPITAL DOTTED LUNATE SIGMA SYMBOL +03FF; C; 037D; # GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL +0400; C; 0450; # CYRILLIC CAPITAL LETTER IE WITH GRAVE +0401; C; 0451; # CYRILLIC CAPITAL LETTER IO +0402; C; 0452; # CYRILLIC CAPITAL LETTER DJE +0403; C; 0453; # CYRILLIC CAPITAL LETTER GJE +0404; C; 0454; # CYRILLIC CAPITAL LETTER UKRAINIAN IE +0405; C; 0455; # CYRILLIC CAPITAL LETTER DZE +0406; C; 0456; # CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I +0407; C; 0457; # CYRILLIC CAPITAL LETTER YI +0408; C; 0458; # CYRILLIC CAPITAL LETTER JE +0409; C; 0459; # CYRILLIC CAPITAL LETTER LJE +040A; C; 045A; # CYRILLIC CAPITAL LETTER NJE +040B; C; 045B; # CYRILLIC CAPITAL LETTER TSHE +040C; C; 045C; # CYRILLIC CAPITAL LETTER KJE +040D; C; 045D; # CYRILLIC CAPITAL LETTER I WITH GRAVE +040E; C; 045E; # CYRILLIC CAPITAL LETTER SHORT U +040F; C; 045F; # CYRILLIC CAPITAL LETTER DZHE +0410; C; 0430; # CYRILLIC CAPITAL LETTER A +0411; C; 0431; # CYRILLIC CAPITAL LETTER BE +0412; C; 0432; # CYRILLIC CAPITAL LETTER VE +0413; C; 0433; # CYRILLIC CAPITAL LETTER GHE +0414; C; 0434; # CYRILLIC CAPITAL LETTER DE +0415; C; 0435; # CYRILLIC CAPITAL LETTER IE +0416; C; 0436; # CYRILLIC CAPITAL LETTER ZHE +0417; C; 0437; # CYRILLIC CAPITAL LETTER ZE +0418; C; 0438; # CYRILLIC CAPITAL LETTER I +0419; C; 0439; # CYRILLIC CAPITAL LETTER SHORT I +041A; C; 043A; # CYRILLIC CAPITAL LETTER KA +041B; C; 043B; # CYRILLIC CAPITAL LETTER EL +041C; C; 043C; # CYRILLIC CAPITAL LETTER EM +041D; C; 043D; # CYRILLIC CAPITAL LETTER EN +041E; C; 043E; # CYRILLIC CAPITAL LETTER O +041F; C; 043F; # CYRILLIC CAPITAL LETTER PE +0420; C; 0440; # CYRILLIC CAPITAL LETTER ER +0421; C; 0441; # CYRILLIC CAPITAL LETTER ES +0422; C; 0442; # CYRILLIC CAPITAL LETTER TE +0423; C; 0443; # CYRILLIC CAPITAL LETTER U +0424; C; 0444; # CYRILLIC CAPITAL LETTER EF +0425; C; 0445; # CYRILLIC CAPITAL LETTER HA +0426; C; 0446; # CYRILLIC CAPITAL LETTER TSE +0427; C; 0447; # CYRILLIC CAPITAL LETTER CHE +0428; C; 0448; # CYRILLIC CAPITAL LETTER SHA +0429; C; 0449; # CYRILLIC CAPITAL LETTER SHCHA +042A; C; 044A; # CYRILLIC CAPITAL LETTER HARD SIGN +042B; C; 044B; # CYRILLIC CAPITAL LETTER YERU +042C; C; 044C; # CYRILLIC CAPITAL LETTER SOFT SIGN +042D; C; 044D; # CYRILLIC CAPITAL LETTER E +042E; C; 044E; # CYRILLIC CAPITAL LETTER YU +042F; C; 044F; # CYRILLIC CAPITAL LETTER YA +0460; C; 0461; # CYRILLIC CAPITAL LETTER OMEGA +0462; C; 0463; # CYRILLIC CAPITAL LETTER YAT +0464; C; 0465; # CYRILLIC CAPITAL LETTER IOTIFIED E +0466; C; 0467; # CYRILLIC CAPITAL LETTER LITTLE YUS +0468; C; 0469; # CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS +046A; C; 046B; # CYRILLIC CAPITAL LETTER BIG YUS +046C; C; 046D; # CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS +046E; C; 046F; # CYRILLIC CAPITAL LETTER KSI +0470; C; 0471; # CYRILLIC CAPITAL LETTER PSI +0472; C; 0473; # CYRILLIC CAPITAL LETTER FITA +0474; C; 0475; # CYRILLIC CAPITAL LETTER IZHITSA +0476; C; 0477; # CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT +0478; C; 0479; # CYRILLIC CAPITAL LETTER UK +047A; C; 047B; # CYRILLIC CAPITAL LETTER ROUND OMEGA +047C; C; 047D; # CYRILLIC CAPITAL LETTER OMEGA WITH TITLO +047E; C; 047F; # CYRILLIC CAPITAL LETTER OT +0480; C; 0481; # CYRILLIC CAPITAL LETTER KOPPA +048A; C; 048B; # CYRILLIC CAPITAL LETTER SHORT I WITH TAIL +048C; C; 048D; # CYRILLIC CAPITAL LETTER SEMISOFT SIGN +048E; C; 048F; # CYRILLIC CAPITAL LETTER ER WITH TICK +0490; C; 0491; # CYRILLIC CAPITAL LETTER GHE WITH UPTURN +0492; C; 0493; # CYRILLIC CAPITAL LETTER GHE WITH STROKE +0494; C; 0495; # CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK +0496; C; 0497; # CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER +0498; C; 0499; # CYRILLIC CAPITAL LETTER ZE WITH DESCENDER +049A; C; 049B; # CYRILLIC CAPITAL LETTER KA WITH DESCENDER +049C; C; 049D; # CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE +049E; C; 049F; # CYRILLIC CAPITAL LETTER KA WITH STROKE +04A0; C; 04A1; # CYRILLIC CAPITAL LETTER BASHKIR KA +04A2; C; 04A3; # CYRILLIC CAPITAL LETTER EN WITH DESCENDER +04A4; C; 04A5; # CYRILLIC CAPITAL LIGATURE EN GHE +04A6; C; 04A7; # CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK +04A8; C; 04A9; # CYRILLIC CAPITAL LETTER ABKHASIAN HA +04AA; C; 04AB; # CYRILLIC CAPITAL LETTER ES WITH DESCENDER +04AC; C; 04AD; # CYRILLIC CAPITAL LETTER TE WITH DESCENDER +04AE; C; 04AF; # CYRILLIC CAPITAL LETTER STRAIGHT U +04B0; C; 04B1; # CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE +04B2; C; 04B3; # CYRILLIC CAPITAL LETTER HA WITH DESCENDER +04B4; C; 04B5; # CYRILLIC CAPITAL LIGATURE TE TSE +04B6; C; 04B7; # CYRILLIC CAPITAL LETTER CHE WITH DESCENDER +04B8; C; 04B9; # CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE +04BA; C; 04BB; # CYRILLIC CAPITAL LETTER SHHA +04BC; C; 04BD; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE +04BE; C; 04BF; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER +04C0; C; 04CF; # CYRILLIC LETTER PALOCHKA +04C1; C; 04C2; # CYRILLIC CAPITAL LETTER ZHE WITH BREVE +04C3; C; 04C4; # CYRILLIC CAPITAL LETTER KA WITH HOOK +04C5; C; 04C6; # CYRILLIC CAPITAL LETTER EL WITH TAIL +04C7; C; 04C8; # CYRILLIC CAPITAL LETTER EN WITH HOOK +04C9; C; 04CA; # CYRILLIC CAPITAL LETTER EN WITH TAIL +04CB; C; 04CC; # CYRILLIC CAPITAL LETTER KHAKASSIAN CHE +04CD; C; 04CE; # CYRILLIC CAPITAL LETTER EM WITH TAIL +04D0; C; 04D1; # CYRILLIC CAPITAL LETTER A WITH BREVE +04D2; C; 04D3; # CYRILLIC CAPITAL LETTER A WITH DIAERESIS +04D4; C; 04D5; # CYRILLIC CAPITAL LIGATURE A IE +04D6; C; 04D7; # CYRILLIC CAPITAL LETTER IE WITH BREVE +04D8; C; 04D9; # CYRILLIC CAPITAL LETTER SCHWA +04DA; C; 04DB; # CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS +04DC; C; 04DD; # CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS +04DE; C; 04DF; # CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS +04E0; C; 04E1; # CYRILLIC CAPITAL LETTER ABKHASIAN DZE +04E2; C; 04E3; # CYRILLIC CAPITAL LETTER I WITH MACRON +04E4; C; 04E5; # CYRILLIC CAPITAL LETTER I WITH DIAERESIS +04E6; C; 04E7; # CYRILLIC CAPITAL LETTER O WITH DIAERESIS +04E8; C; 04E9; # CYRILLIC CAPITAL LETTER BARRED O +04EA; C; 04EB; # CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS +04EC; C; 04ED; # CYRILLIC CAPITAL LETTER E WITH DIAERESIS +04EE; C; 04EF; # CYRILLIC CAPITAL LETTER U WITH MACRON +04F0; C; 04F1; # CYRILLIC CAPITAL LETTER U WITH DIAERESIS +04F2; C; 04F3; # CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE +04F4; C; 04F5; # CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS +04F6; C; 04F7; # CYRILLIC CAPITAL LETTER GHE WITH DESCENDER +04F8; C; 04F9; # CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS +04FA; C; 04FB; # CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK +04FC; C; 04FD; # CYRILLIC CAPITAL LETTER HA WITH HOOK +04FE; C; 04FF; # CYRILLIC CAPITAL LETTER HA WITH STROKE +0500; C; 0501; # CYRILLIC CAPITAL LETTER KOMI DE +0502; C; 0503; # CYRILLIC CAPITAL LETTER KOMI DJE +0504; C; 0505; # CYRILLIC CAPITAL LETTER KOMI ZJE +0506; C; 0507; # CYRILLIC CAPITAL LETTER KOMI DZJE +0508; C; 0509; # CYRILLIC CAPITAL LETTER KOMI LJE +050A; C; 050B; # CYRILLIC CAPITAL LETTER KOMI NJE +050C; C; 050D; # CYRILLIC CAPITAL LETTER KOMI SJE +050E; C; 050F; # CYRILLIC CAPITAL LETTER KOMI TJE +0510; C; 0511; # CYRILLIC CAPITAL LETTER REVERSED ZE +0512; C; 0513; # CYRILLIC CAPITAL LETTER EL WITH HOOK +0514; C; 0515; # CYRILLIC CAPITAL LETTER LHA +0516; C; 0517; # CYRILLIC CAPITAL LETTER RHA +0518; C; 0519; # CYRILLIC CAPITAL LETTER YAE +051A; C; 051B; # CYRILLIC CAPITAL LETTER QA +051C; C; 051D; # CYRILLIC CAPITAL LETTER WE +051E; C; 051F; # CYRILLIC CAPITAL LETTER ALEUT KA +0520; C; 0521; # CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK +0522; C; 0523; # CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK +0524; C; 0525; # CYRILLIC CAPITAL LETTER PE WITH DESCENDER +0526; C; 0527; # CYRILLIC CAPITAL LETTER SHHA WITH DESCENDER +0528; C; 0529; # CYRILLIC CAPITAL LETTER EN WITH LEFT HOOK +052A; C; 052B; # CYRILLIC CAPITAL LETTER DZZHE +052C; C; 052D; # CYRILLIC CAPITAL LETTER DCHE +052E; C; 052F; # CYRILLIC CAPITAL LETTER EL WITH DESCENDER +0531; C; 0561; # ARMENIAN CAPITAL LETTER AYB +0532; C; 0562; # ARMENIAN CAPITAL LETTER BEN +0533; C; 0563; # ARMENIAN CAPITAL LETTER GIM +0534; C; 0564; # ARMENIAN CAPITAL LETTER DA +0535; C; 0565; # ARMENIAN CAPITAL LETTER ECH +0536; C; 0566; # ARMENIAN CAPITAL LETTER ZA +0537; C; 0567; # ARMENIAN CAPITAL LETTER EH +0538; C; 0568; # ARMENIAN CAPITAL LETTER ET +0539; C; 0569; # ARMENIAN CAPITAL LETTER TO +053A; C; 056A; # ARMENIAN CAPITAL LETTER ZHE +053B; C; 056B; # ARMENIAN CAPITAL LETTER INI +053C; C; 056C; # ARMENIAN CAPITAL LETTER LIWN +053D; C; 056D; # ARMENIAN CAPITAL LETTER XEH +053E; C; 056E; # ARMENIAN CAPITAL LETTER CA +053F; C; 056F; # ARMENIAN CAPITAL LETTER KEN +0540; C; 0570; # ARMENIAN CAPITAL LETTER HO +0541; C; 0571; # ARMENIAN CAPITAL LETTER JA +0542; C; 0572; # ARMENIAN CAPITAL LETTER GHAD +0543; C; 0573; # ARMENIAN CAPITAL LETTER CHEH +0544; C; 0574; # ARMENIAN CAPITAL LETTER MEN +0545; C; 0575; # ARMENIAN CAPITAL LETTER YI +0546; C; 0576; # ARMENIAN CAPITAL LETTER NOW +0547; C; 0577; # ARMENIAN CAPITAL LETTER SHA +0548; C; 0578; # ARMENIAN CAPITAL LETTER VO +0549; C; 0579; # ARMENIAN CAPITAL LETTER CHA +054A; C; 057A; # ARMENIAN CAPITAL LETTER PEH +054B; C; 057B; # ARMENIAN CAPITAL LETTER JHEH +054C; C; 057C; # ARMENIAN CAPITAL LETTER RA +054D; C; 057D; # ARMENIAN CAPITAL LETTER SEH +054E; C; 057E; # ARMENIAN CAPITAL LETTER VEW +054F; C; 057F; # ARMENIAN CAPITAL LETTER TIWN +0550; C; 0580; # ARMENIAN CAPITAL LETTER REH +0551; C; 0581; # ARMENIAN CAPITAL LETTER CO +0552; C; 0582; # ARMENIAN CAPITAL LETTER YIWN +0553; C; 0583; # ARMENIAN CAPITAL LETTER PIWR +0554; C; 0584; # ARMENIAN CAPITAL LETTER KEH +0555; C; 0585; # ARMENIAN CAPITAL LETTER OH +0556; C; 0586; # ARMENIAN CAPITAL LETTER FEH +0587; F; 0565 0582; # ARMENIAN SMALL LIGATURE ECH YIWN +10A0; C; 2D00; # GEORGIAN CAPITAL LETTER AN +10A1; C; 2D01; # GEORGIAN CAPITAL LETTER BAN +10A2; C; 2D02; # GEORGIAN CAPITAL LETTER GAN +10A3; C; 2D03; # GEORGIAN CAPITAL LETTER DON +10A4; C; 2D04; # GEORGIAN CAPITAL LETTER EN +10A5; C; 2D05; # GEORGIAN CAPITAL LETTER VIN +10A6; C; 2D06; # GEORGIAN CAPITAL LETTER ZEN +10A7; C; 2D07; # GEORGIAN CAPITAL LETTER TAN +10A8; C; 2D08; # GEORGIAN CAPITAL LETTER IN +10A9; C; 2D09; # GEORGIAN CAPITAL LETTER KAN +10AA; C; 2D0A; # GEORGIAN CAPITAL LETTER LAS +10AB; C; 2D0B; # GEORGIAN CAPITAL LETTER MAN +10AC; C; 2D0C; # GEORGIAN CAPITAL LETTER NAR +10AD; C; 2D0D; # GEORGIAN CAPITAL LETTER ON +10AE; C; 2D0E; # GEORGIAN CAPITAL LETTER PAR +10AF; C; 2D0F; # GEORGIAN CAPITAL LETTER ZHAR +10B0; C; 2D10; # GEORGIAN CAPITAL LETTER RAE +10B1; C; 2D11; # GEORGIAN CAPITAL LETTER SAN +10B2; C; 2D12; # GEORGIAN CAPITAL LETTER TAR +10B3; C; 2D13; # GEORGIAN CAPITAL LETTER UN +10B4; C; 2D14; # GEORGIAN CAPITAL LETTER PHAR +10B5; C; 2D15; # GEORGIAN CAPITAL LETTER KHAR +10B6; C; 2D16; # GEORGIAN CAPITAL LETTER GHAN +10B7; C; 2D17; # GEORGIAN CAPITAL LETTER QAR +10B8; C; 2D18; # GEORGIAN CAPITAL LETTER SHIN +10B9; C; 2D19; # GEORGIAN CAPITAL LETTER CHIN +10BA; C; 2D1A; # GEORGIAN CAPITAL LETTER CAN +10BB; C; 2D1B; # GEORGIAN CAPITAL LETTER JIL +10BC; C; 2D1C; # GEORGIAN CAPITAL LETTER CIL +10BD; C; 2D1D; # GEORGIAN CAPITAL LETTER CHAR +10BE; C; 2D1E; # GEORGIAN CAPITAL LETTER XAN +10BF; C; 2D1F; # GEORGIAN CAPITAL LETTER JHAN +10C0; C; 2D20; # GEORGIAN CAPITAL LETTER HAE +10C1; C; 2D21; # GEORGIAN CAPITAL LETTER HE +10C2; C; 2D22; # GEORGIAN CAPITAL LETTER HIE +10C3; C; 2D23; # GEORGIAN CAPITAL LETTER WE +10C4; C; 2D24; # GEORGIAN CAPITAL LETTER HAR +10C5; C; 2D25; # GEORGIAN CAPITAL LETTER HOE +10C7; C; 2D27; # GEORGIAN CAPITAL LETTER YN +10CD; C; 2D2D; # GEORGIAN CAPITAL LETTER AEN +13F8; C; 13F0; # CHEROKEE SMALL LETTER YE +13F9; C; 13F1; # CHEROKEE SMALL LETTER YI +13FA; C; 13F2; # CHEROKEE SMALL LETTER YO +13FB; C; 13F3; # CHEROKEE SMALL LETTER YU +13FC; C; 13F4; # CHEROKEE SMALL LETTER YV +13FD; C; 13F5; # CHEROKEE SMALL LETTER MV +1C80; C; 0432; # CYRILLIC SMALL LETTER ROUNDED VE +1C81; C; 0434; # CYRILLIC SMALL LETTER LONG-LEGGED DE +1C82; C; 043E; # CYRILLIC SMALL LETTER NARROW O +1C83; C; 0441; # CYRILLIC SMALL LETTER WIDE ES +1C84; C; 0442; # CYRILLIC SMALL LETTER TALL TE +1C85; C; 0442; # CYRILLIC SMALL LETTER THREE-LEGGED TE +1C86; C; 044A; # CYRILLIC SMALL LETTER TALL HARD SIGN +1C87; C; 0463; # CYRILLIC SMALL LETTER TALL YAT +1C88; C; A64B; # CYRILLIC SMALL LETTER UNBLENDED UK +1E00; C; 1E01; # LATIN CAPITAL LETTER A WITH RING BELOW +1E02; C; 1E03; # LATIN CAPITAL LETTER B WITH DOT ABOVE +1E04; C; 1E05; # LATIN CAPITAL LETTER B WITH DOT BELOW +1E06; C; 1E07; # LATIN CAPITAL LETTER B WITH LINE BELOW +1E08; C; 1E09; # LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE +1E0A; C; 1E0B; # LATIN CAPITAL LETTER D WITH DOT ABOVE +1E0C; C; 1E0D; # LATIN CAPITAL LETTER D WITH DOT BELOW +1E0E; C; 1E0F; # LATIN CAPITAL LETTER D WITH LINE BELOW +1E10; C; 1E11; # LATIN CAPITAL LETTER D WITH CEDILLA +1E12; C; 1E13; # LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW +1E14; C; 1E15; # LATIN CAPITAL LETTER E WITH MACRON AND GRAVE +1E16; C; 1E17; # LATIN CAPITAL LETTER E WITH MACRON AND ACUTE +1E18; C; 1E19; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW +1E1A; C; 1E1B; # LATIN CAPITAL LETTER E WITH TILDE BELOW +1E1C; C; 1E1D; # LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE +1E1E; C; 1E1F; # LATIN CAPITAL LETTER F WITH DOT ABOVE +1E20; C; 1E21; # LATIN CAPITAL LETTER G WITH MACRON +1E22; C; 1E23; # LATIN CAPITAL LETTER H WITH DOT ABOVE +1E24; C; 1E25; # LATIN CAPITAL LETTER H WITH DOT BELOW +1E26; C; 1E27; # LATIN CAPITAL LETTER H WITH DIAERESIS +1E28; C; 1E29; # LATIN CAPITAL LETTER H WITH CEDILLA +1E2A; C; 1E2B; # LATIN CAPITAL LETTER H WITH BREVE BELOW +1E2C; C; 1E2D; # LATIN CAPITAL LETTER I WITH TILDE BELOW +1E2E; C; 1E2F; # LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE +1E30; C; 1E31; # LATIN CAPITAL LETTER K WITH ACUTE +1E32; C; 1E33; # LATIN CAPITAL LETTER K WITH DOT BELOW +1E34; C; 1E35; # LATIN CAPITAL LETTER K WITH LINE BELOW +1E36; C; 1E37; # LATIN CAPITAL LETTER L WITH DOT BELOW +1E38; C; 1E39; # LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON +1E3A; C; 1E3B; # LATIN CAPITAL LETTER L WITH LINE BELOW +1E3C; C; 1E3D; # LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW +1E3E; C; 1E3F; # LATIN CAPITAL LETTER M WITH ACUTE +1E40; C; 1E41; # LATIN CAPITAL LETTER M WITH DOT ABOVE +1E42; C; 1E43; # LATIN CAPITAL LETTER M WITH DOT BELOW +1E44; C; 1E45; # LATIN CAPITAL LETTER N WITH DOT ABOVE +1E46; C; 1E47; # LATIN CAPITAL LETTER N WITH DOT BELOW +1E48; C; 1E49; # LATIN CAPITAL LETTER N WITH LINE BELOW +1E4A; C; 1E4B; # LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW +1E4C; C; 1E4D; # LATIN CAPITAL LETTER O WITH TILDE AND ACUTE +1E4E; C; 1E4F; # LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS +1E50; C; 1E51; # LATIN CAPITAL LETTER O WITH MACRON AND GRAVE +1E52; C; 1E53; # LATIN CAPITAL LETTER O WITH MACRON AND ACUTE +1E54; C; 1E55; # LATIN CAPITAL LETTER P WITH ACUTE +1E56; C; 1E57; # LATIN CAPITAL LETTER P WITH DOT ABOVE +1E58; C; 1E59; # LATIN CAPITAL LETTER R WITH DOT ABOVE +1E5A; C; 1E5B; # LATIN CAPITAL LETTER R WITH DOT BELOW +1E5C; C; 1E5D; # LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON +1E5E; C; 1E5F; # LATIN CAPITAL LETTER R WITH LINE BELOW +1E60; C; 1E61; # LATIN CAPITAL LETTER S WITH DOT ABOVE +1E62; C; 1E63; # LATIN CAPITAL LETTER S WITH DOT BELOW +1E64; C; 1E65; # LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE +1E66; C; 1E67; # LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE +1E68; C; 1E69; # LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE +1E6A; C; 1E6B; # LATIN CAPITAL LETTER T WITH DOT ABOVE +1E6C; C; 1E6D; # LATIN CAPITAL LETTER T WITH DOT BELOW +1E6E; C; 1E6F; # LATIN CAPITAL LETTER T WITH LINE BELOW +1E70; C; 1E71; # LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW +1E72; C; 1E73; # LATIN CAPITAL LETTER U WITH DIAERESIS BELOW +1E74; C; 1E75; # LATIN CAPITAL LETTER U WITH TILDE BELOW +1E76; C; 1E77; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW +1E78; C; 1E79; # LATIN CAPITAL LETTER U WITH TILDE AND ACUTE +1E7A; C; 1E7B; # LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS +1E7C; C; 1E7D; # LATIN CAPITAL LETTER V WITH TILDE +1E7E; C; 1E7F; # LATIN CAPITAL LETTER V WITH DOT BELOW +1E80; C; 1E81; # LATIN CAPITAL LETTER W WITH GRAVE +1E82; C; 1E83; # LATIN CAPITAL LETTER W WITH ACUTE +1E84; C; 1E85; # LATIN CAPITAL LETTER W WITH DIAERESIS +1E86; C; 1E87; # LATIN CAPITAL LETTER W WITH DOT ABOVE +1E88; C; 1E89; # LATIN CAPITAL LETTER W WITH DOT BELOW +1E8A; C; 1E8B; # LATIN CAPITAL LETTER X WITH DOT ABOVE +1E8C; C; 1E8D; # LATIN CAPITAL LETTER X WITH DIAERESIS +1E8E; C; 1E8F; # LATIN CAPITAL LETTER Y WITH DOT ABOVE +1E90; C; 1E91; # LATIN CAPITAL LETTER Z WITH CIRCUMFLEX +1E92; C; 1E93; # LATIN CAPITAL LETTER Z WITH DOT BELOW +1E94; C; 1E95; # LATIN CAPITAL LETTER Z WITH LINE BELOW +1E96; F; 0068 0331; # LATIN SMALL LETTER H WITH LINE BELOW +1E97; F; 0074 0308; # LATIN SMALL LETTER T WITH DIAERESIS +1E98; F; 0077 030A; # LATIN SMALL LETTER W WITH RING ABOVE +1E99; F; 0079 030A; # LATIN SMALL LETTER Y WITH RING ABOVE +1E9A; F; 0061 02BE; # LATIN SMALL LETTER A WITH RIGHT HALF RING +1E9B; C; 1E61; # LATIN SMALL LETTER LONG S WITH DOT ABOVE +1E9E; F; 0073 0073; # LATIN CAPITAL LETTER SHARP S +1E9E; S; 00DF; # LATIN CAPITAL LETTER SHARP S +1EA0; C; 1EA1; # LATIN CAPITAL LETTER A WITH DOT BELOW +1EA2; C; 1EA3; # LATIN CAPITAL LETTER A WITH HOOK ABOVE +1EA4; C; 1EA5; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE +1EA6; C; 1EA7; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE +1EA8; C; 1EA9; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE +1EAA; C; 1EAB; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE +1EAC; C; 1EAD; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW +1EAE; C; 1EAF; # LATIN CAPITAL LETTER A WITH BREVE AND ACUTE +1EB0; C; 1EB1; # LATIN CAPITAL LETTER A WITH BREVE AND GRAVE +1EB2; C; 1EB3; # LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE +1EB4; C; 1EB5; # LATIN CAPITAL LETTER A WITH BREVE AND TILDE +1EB6; C; 1EB7; # LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW +1EB8; C; 1EB9; # LATIN CAPITAL LETTER E WITH DOT BELOW +1EBA; C; 1EBB; # LATIN CAPITAL LETTER E WITH HOOK ABOVE +1EBC; C; 1EBD; # LATIN CAPITAL LETTER E WITH TILDE +1EBE; C; 1EBF; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE +1EC0; C; 1EC1; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE +1EC2; C; 1EC3; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE +1EC4; C; 1EC5; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE +1EC6; C; 1EC7; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW +1EC8; C; 1EC9; # LATIN CAPITAL LETTER I WITH HOOK ABOVE +1ECA; C; 1ECB; # LATIN CAPITAL LETTER I WITH DOT BELOW +1ECC; C; 1ECD; # LATIN CAPITAL LETTER O WITH DOT BELOW +1ECE; C; 1ECF; # LATIN CAPITAL LETTER O WITH HOOK ABOVE +1ED0; C; 1ED1; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE +1ED2; C; 1ED3; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE +1ED4; C; 1ED5; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE +1ED6; C; 1ED7; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE +1ED8; C; 1ED9; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW +1EDA; C; 1EDB; # LATIN CAPITAL LETTER O WITH HORN AND ACUTE +1EDC; C; 1EDD; # LATIN CAPITAL LETTER O WITH HORN AND GRAVE +1EDE; C; 1EDF; # LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE +1EE0; C; 1EE1; # LATIN CAPITAL LETTER O WITH HORN AND TILDE +1EE2; C; 1EE3; # LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW +1EE4; C; 1EE5; # LATIN CAPITAL LETTER U WITH DOT BELOW +1EE6; C; 1EE7; # LATIN CAPITAL LETTER U WITH HOOK ABOVE +1EE8; C; 1EE9; # LATIN CAPITAL LETTER U WITH HORN AND ACUTE +1EEA; C; 1EEB; # LATIN CAPITAL LETTER U WITH HORN AND GRAVE +1EEC; C; 1EED; # LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE +1EEE; C; 1EEF; # LATIN CAPITAL LETTER U WITH HORN AND TILDE +1EF0; C; 1EF1; # LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW +1EF2; C; 1EF3; # LATIN CAPITAL LETTER Y WITH GRAVE +1EF4; C; 1EF5; # LATIN CAPITAL LETTER Y WITH DOT BELOW +1EF6; C; 1EF7; # LATIN CAPITAL LETTER Y WITH HOOK ABOVE +1EF8; C; 1EF9; # LATIN CAPITAL LETTER Y WITH TILDE +1EFA; C; 1EFB; # LATIN CAPITAL LETTER MIDDLE-WELSH LL +1EFC; C; 1EFD; # LATIN CAPITAL LETTER MIDDLE-WELSH V +1EFE; C; 1EFF; # LATIN CAPITAL LETTER Y WITH LOOP +1F08; C; 1F00; # GREEK CAPITAL LETTER ALPHA WITH PSILI +1F09; C; 1F01; # GREEK CAPITAL LETTER ALPHA WITH DASIA +1F0A; C; 1F02; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA +1F0B; C; 1F03; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA +1F0C; C; 1F04; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA +1F0D; C; 1F05; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA +1F0E; C; 1F06; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI +1F0F; C; 1F07; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI +1F18; C; 1F10; # GREEK CAPITAL LETTER EPSILON WITH PSILI +1F19; C; 1F11; # GREEK CAPITAL LETTER EPSILON WITH DASIA +1F1A; C; 1F12; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND VARIA +1F1B; C; 1F13; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND VARIA +1F1C; C; 1F14; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND OXIA +1F1D; C; 1F15; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA +1F28; C; 1F20; # GREEK CAPITAL LETTER ETA WITH PSILI +1F29; C; 1F21; # GREEK CAPITAL LETTER ETA WITH DASIA +1F2A; C; 1F22; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA +1F2B; C; 1F23; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA +1F2C; C; 1F24; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA +1F2D; C; 1F25; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA +1F2E; C; 1F26; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI +1F2F; C; 1F27; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI +1F38; C; 1F30; # GREEK CAPITAL LETTER IOTA WITH PSILI +1F39; C; 1F31; # GREEK CAPITAL LETTER IOTA WITH DASIA +1F3A; C; 1F32; # GREEK CAPITAL LETTER IOTA WITH PSILI AND VARIA +1F3B; C; 1F33; # GREEK CAPITAL LETTER IOTA WITH DASIA AND VARIA +1F3C; C; 1F34; # GREEK CAPITAL LETTER IOTA WITH PSILI AND OXIA +1F3D; C; 1F35; # GREEK CAPITAL LETTER IOTA WITH DASIA AND OXIA +1F3E; C; 1F36; # GREEK CAPITAL LETTER IOTA WITH PSILI AND PERISPOMENI +1F3F; C; 1F37; # GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI +1F48; C; 1F40; # GREEK CAPITAL LETTER OMICRON WITH PSILI +1F49; C; 1F41; # GREEK CAPITAL LETTER OMICRON WITH DASIA +1F4A; C; 1F42; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND VARIA +1F4B; C; 1F43; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND VARIA +1F4C; C; 1F44; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND OXIA +1F4D; C; 1F45; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA +1F50; F; 03C5 0313; # GREEK SMALL LETTER UPSILON WITH PSILI +1F52; F; 03C5 0313 0300; # GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA +1F54; F; 03C5 0313 0301; # GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA +1F56; F; 03C5 0313 0342; # GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI +1F59; C; 1F51; # GREEK CAPITAL LETTER UPSILON WITH DASIA +1F5B; C; 1F53; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA +1F5D; C; 1F55; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA +1F5F; C; 1F57; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F68; C; 1F60; # GREEK CAPITAL LETTER OMEGA WITH PSILI +1F69; C; 1F61; # GREEK CAPITAL LETTER OMEGA WITH DASIA +1F6A; C; 1F62; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA +1F6B; C; 1F63; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA +1F6C; C; 1F64; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA +1F6D; C; 1F65; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA +1F6E; C; 1F66; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI +1F6F; C; 1F67; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI +1F80; F; 1F00 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI +1F81; F; 1F01 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI +1F82; F; 1F02 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI +1F83; F; 1F03 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI +1F84; F; 1F04 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI +1F85; F; 1F05 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI +1F86; F; 1F06 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI +1F87; F; 1F07 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1F88; F; 1F00 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI +1F88; S; 1F80; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI +1F89; F; 1F01 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI +1F89; S; 1F81; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI +1F8A; F; 1F02 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1F8A; S; 1F82; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1F8B; F; 1F03 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1F8B; S; 1F83; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1F8C; F; 1F04 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1F8C; S; 1F84; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1F8D; F; 1F05 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1F8D; S; 1F85; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1F8E; F; 1F06 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1F8E; S; 1F86; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1F8F; F; 1F07 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1F8F; S; 1F87; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1F90; F; 1F20 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI +1F91; F; 1F21 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI +1F92; F; 1F22 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI +1F93; F; 1F23 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI +1F94; F; 1F24 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI +1F95; F; 1F25 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI +1F96; F; 1F26 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI +1F97; F; 1F27 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1F98; F; 1F20 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI +1F98; S; 1F90; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI +1F99; F; 1F21 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI +1F99; S; 1F91; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI +1F9A; F; 1F22 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1F9A; S; 1F92; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1F9B; F; 1F23 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1F9B; S; 1F93; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1F9C; F; 1F24 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1F9C; S; 1F94; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1F9D; F; 1F25 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1F9D; S; 1F95; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1F9E; F; 1F26 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1F9E; S; 1F96; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1F9F; F; 1F27 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1F9F; S; 1F97; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1FA0; F; 1F60 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI +1FA1; F; 1F61 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI +1FA2; F; 1F62 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI +1FA3; F; 1F63 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI +1FA4; F; 1F64 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI +1FA5; F; 1F65 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI +1FA6; F; 1F66 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI +1FA7; F; 1F67 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1FA8; F; 1F60 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI +1FA8; S; 1FA0; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI +1FA9; F; 1F61 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI +1FA9; S; 1FA1; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI +1FAA; F; 1F62 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1FAA; S; 1FA2; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1FAB; F; 1F63 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1FAB; S; 1FA3; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1FAC; F; 1F64 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1FAC; S; 1FA4; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1FAD; F; 1F65 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1FAD; S; 1FA5; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1FAE; F; 1F66 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1FAE; S; 1FA6; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1FAF; F; 1F67 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1FAF; S; 1FA7; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1FB2; F; 1F70 03B9; # GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI +1FB3; F; 03B1 03B9; # GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI +1FB4; F; 03AC 03B9; # GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FB6; F; 03B1 0342; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI +1FB7; F; 03B1 0342 03B9; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI +1FB8; C; 1FB0; # GREEK CAPITAL LETTER ALPHA WITH VRACHY +1FB9; C; 1FB1; # GREEK CAPITAL LETTER ALPHA WITH MACRON +1FBA; C; 1F70; # GREEK CAPITAL LETTER ALPHA WITH VARIA +1FBB; C; 1F71; # GREEK CAPITAL LETTER ALPHA WITH OXIA +1FBC; F; 03B1 03B9; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FBC; S; 1FB3; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FBE; C; 03B9; # GREEK PROSGEGRAMMENI +1FC2; F; 1F74 03B9; # GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI +1FC3; F; 03B7 03B9; # GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI +1FC4; F; 03AE 03B9; # GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC6; F; 03B7 0342; # GREEK SMALL LETTER ETA WITH PERISPOMENI +1FC7; F; 03B7 0342 03B9; # GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI +1FC8; C; 1F72; # GREEK CAPITAL LETTER EPSILON WITH VARIA +1FC9; C; 1F73; # GREEK CAPITAL LETTER EPSILON WITH OXIA +1FCA; C; 1F74; # GREEK CAPITAL LETTER ETA WITH VARIA +1FCB; C; 1F75; # GREEK CAPITAL LETTER ETA WITH OXIA +1FCC; F; 03B7 03B9; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FCC; S; 1FC3; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FD2; F; 03B9 0308 0300; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA +1FD3; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD6; F; 03B9 0342; # GREEK SMALL LETTER IOTA WITH PERISPOMENI +1FD7; F; 03B9 0308 0342; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI +1FD8; C; 1FD0; # GREEK CAPITAL LETTER IOTA WITH VRACHY +1FD9; C; 1FD1; # GREEK CAPITAL LETTER IOTA WITH MACRON +1FDA; C; 1F76; # GREEK CAPITAL LETTER IOTA WITH VARIA +1FDB; C; 1F77; # GREEK CAPITAL LETTER IOTA WITH OXIA +1FE2; F; 03C5 0308 0300; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA +1FE3; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA +1FE4; F; 03C1 0313; # GREEK SMALL LETTER RHO WITH PSILI +1FE6; F; 03C5 0342; # GREEK SMALL LETTER UPSILON WITH PERISPOMENI +1FE7; F; 03C5 0308 0342; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI +1FE8; C; 1FE0; # GREEK CAPITAL LETTER UPSILON WITH VRACHY +1FE9; C; 1FE1; # GREEK CAPITAL LETTER UPSILON WITH MACRON +1FEA; C; 1F7A; # GREEK CAPITAL LETTER UPSILON WITH VARIA +1FEB; C; 1F7B; # GREEK CAPITAL LETTER UPSILON WITH OXIA +1FEC; C; 1FE5; # GREEK CAPITAL LETTER RHO WITH DASIA +1FF2; F; 1F7C 03B9; # GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI +1FF3; F; 03C9 03B9; # GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI +1FF4; F; 03CE 03B9; # GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF6; F; 03C9 0342; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI +1FF7; F; 03C9 0342 03B9; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI +1FF8; C; 1F78; # GREEK CAPITAL LETTER OMICRON WITH VARIA +1FF9; C; 1F79; # GREEK CAPITAL LETTER OMICRON WITH OXIA +1FFA; C; 1F7C; # GREEK CAPITAL LETTER OMEGA WITH VARIA +1FFB; C; 1F7D; # GREEK CAPITAL LETTER OMEGA WITH OXIA +1FFC; F; 03C9 03B9; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +1FFC; S; 1FF3; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +2126; C; 03C9; # OHM SIGN +212A; C; 006B; # KELVIN SIGN +212B; C; 00E5; # ANGSTROM SIGN +2132; C; 214E; # TURNED CAPITAL F +2160; C; 2170; # ROMAN NUMERAL ONE +2161; C; 2171; # ROMAN NUMERAL TWO +2162; C; 2172; # ROMAN NUMERAL THREE +2163; C; 2173; # ROMAN NUMERAL FOUR +2164; C; 2174; # ROMAN NUMERAL FIVE +2165; C; 2175; # ROMAN NUMERAL SIX +2166; C; 2176; # ROMAN NUMERAL SEVEN +2167; C; 2177; # ROMAN NUMERAL EIGHT +2168; C; 2178; # ROMAN NUMERAL NINE +2169; C; 2179; # ROMAN NUMERAL TEN +216A; C; 217A; # ROMAN NUMERAL ELEVEN +216B; C; 217B; # ROMAN NUMERAL TWELVE +216C; C; 217C; # ROMAN NUMERAL FIFTY +216D; C; 217D; # ROMAN NUMERAL ONE HUNDRED +216E; C; 217E; # ROMAN NUMERAL FIVE HUNDRED +216F; C; 217F; # ROMAN NUMERAL ONE THOUSAND +2183; C; 2184; # ROMAN NUMERAL REVERSED ONE HUNDRED +24B6; C; 24D0; # CIRCLED LATIN CAPITAL LETTER A +24B7; C; 24D1; # CIRCLED LATIN CAPITAL LETTER B +24B8; C; 24D2; # CIRCLED LATIN CAPITAL LETTER C +24B9; C; 24D3; # CIRCLED LATIN CAPITAL LETTER D +24BA; C; 24D4; # CIRCLED LATIN CAPITAL LETTER E +24BB; C; 24D5; # CIRCLED LATIN CAPITAL LETTER F +24BC; C; 24D6; # CIRCLED LATIN CAPITAL LETTER G +24BD; C; 24D7; # CIRCLED LATIN CAPITAL LETTER H +24BE; C; 24D8; # CIRCLED LATIN CAPITAL LETTER I +24BF; C; 24D9; # CIRCLED LATIN CAPITAL LETTER J +24C0; C; 24DA; # CIRCLED LATIN CAPITAL LETTER K +24C1; C; 24DB; # CIRCLED LATIN CAPITAL LETTER L +24C2; C; 24DC; # CIRCLED LATIN CAPITAL LETTER M +24C3; C; 24DD; # CIRCLED LATIN CAPITAL LETTER N +24C4; C; 24DE; # CIRCLED LATIN CAPITAL LETTER O +24C5; C; 24DF; # CIRCLED LATIN CAPITAL LETTER P +24C6; C; 24E0; # CIRCLED LATIN CAPITAL LETTER Q +24C7; C; 24E1; # CIRCLED LATIN CAPITAL LETTER R +24C8; C; 24E2; # CIRCLED LATIN CAPITAL LETTER S +24C9; C; 24E3; # CIRCLED LATIN CAPITAL LETTER T +24CA; C; 24E4; # CIRCLED LATIN CAPITAL LETTER U +24CB; C; 24E5; # CIRCLED LATIN CAPITAL LETTER V +24CC; C; 24E6; # CIRCLED LATIN CAPITAL LETTER W +24CD; C; 24E7; # CIRCLED LATIN CAPITAL LETTER X +24CE; C; 24E8; # CIRCLED LATIN CAPITAL LETTER Y +24CF; C; 24E9; # CIRCLED LATIN CAPITAL LETTER Z +2C00; C; 2C30; # GLAGOLITIC CAPITAL LETTER AZU +2C01; C; 2C31; # GLAGOLITIC CAPITAL LETTER BUKY +2C02; C; 2C32; # GLAGOLITIC CAPITAL LETTER VEDE +2C03; C; 2C33; # GLAGOLITIC CAPITAL LETTER GLAGOLI +2C04; C; 2C34; # GLAGOLITIC CAPITAL LETTER DOBRO +2C05; C; 2C35; # GLAGOLITIC CAPITAL LETTER YESTU +2C06; C; 2C36; # GLAGOLITIC CAPITAL LETTER ZHIVETE +2C07; C; 2C37; # GLAGOLITIC CAPITAL LETTER DZELO +2C08; C; 2C38; # GLAGOLITIC CAPITAL LETTER ZEMLJA +2C09; C; 2C39; # GLAGOLITIC CAPITAL LETTER IZHE +2C0A; C; 2C3A; # GLAGOLITIC CAPITAL LETTER INITIAL IZHE +2C0B; C; 2C3B; # GLAGOLITIC CAPITAL LETTER I +2C0C; C; 2C3C; # GLAGOLITIC CAPITAL LETTER DJERVI +2C0D; C; 2C3D; # GLAGOLITIC CAPITAL LETTER KAKO +2C0E; C; 2C3E; # GLAGOLITIC CAPITAL LETTER LJUDIJE +2C0F; C; 2C3F; # GLAGOLITIC CAPITAL LETTER MYSLITE +2C10; C; 2C40; # GLAGOLITIC CAPITAL LETTER NASHI +2C11; C; 2C41; # GLAGOLITIC CAPITAL LETTER ONU +2C12; C; 2C42; # GLAGOLITIC CAPITAL LETTER POKOJI +2C13; C; 2C43; # GLAGOLITIC CAPITAL LETTER RITSI +2C14; C; 2C44; # GLAGOLITIC CAPITAL LETTER SLOVO +2C15; C; 2C45; # GLAGOLITIC CAPITAL LETTER TVRIDO +2C16; C; 2C46; # GLAGOLITIC CAPITAL LETTER UKU +2C17; C; 2C47; # GLAGOLITIC CAPITAL LETTER FRITU +2C18; C; 2C48; # GLAGOLITIC CAPITAL LETTER HERU +2C19; C; 2C49; # GLAGOLITIC CAPITAL LETTER OTU +2C1A; C; 2C4A; # GLAGOLITIC CAPITAL LETTER PE +2C1B; C; 2C4B; # GLAGOLITIC CAPITAL LETTER SHTA +2C1C; C; 2C4C; # GLAGOLITIC CAPITAL LETTER TSI +2C1D; C; 2C4D; # GLAGOLITIC CAPITAL LETTER CHRIVI +2C1E; C; 2C4E; # GLAGOLITIC CAPITAL LETTER SHA +2C1F; C; 2C4F; # GLAGOLITIC CAPITAL LETTER YERU +2C20; C; 2C50; # GLAGOLITIC CAPITAL LETTER YERI +2C21; C; 2C51; # GLAGOLITIC CAPITAL LETTER YATI +2C22; C; 2C52; # GLAGOLITIC CAPITAL LETTER SPIDERY HA +2C23; C; 2C53; # GLAGOLITIC CAPITAL LETTER YU +2C24; C; 2C54; # GLAGOLITIC CAPITAL LETTER SMALL YUS +2C25; C; 2C55; # GLAGOLITIC CAPITAL LETTER SMALL YUS WITH TAIL +2C26; C; 2C56; # GLAGOLITIC CAPITAL LETTER YO +2C27; C; 2C57; # GLAGOLITIC CAPITAL LETTER IOTATED SMALL YUS +2C28; C; 2C58; # GLAGOLITIC CAPITAL LETTER BIG YUS +2C29; C; 2C59; # GLAGOLITIC CAPITAL LETTER IOTATED BIG YUS +2C2A; C; 2C5A; # GLAGOLITIC CAPITAL LETTER FITA +2C2B; C; 2C5B; # GLAGOLITIC CAPITAL LETTER IZHITSA +2C2C; C; 2C5C; # GLAGOLITIC CAPITAL LETTER SHTAPIC +2C2D; C; 2C5D; # GLAGOLITIC CAPITAL LETTER TROKUTASTI A +2C2E; C; 2C5E; # GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE +2C60; C; 2C61; # LATIN CAPITAL LETTER L WITH DOUBLE BAR +2C62; C; 026B; # LATIN CAPITAL LETTER L WITH MIDDLE TILDE +2C63; C; 1D7D; # LATIN CAPITAL LETTER P WITH STROKE +2C64; C; 027D; # LATIN CAPITAL LETTER R WITH TAIL +2C67; C; 2C68; # LATIN CAPITAL LETTER H WITH DESCENDER +2C69; C; 2C6A; # LATIN CAPITAL LETTER K WITH DESCENDER +2C6B; C; 2C6C; # LATIN CAPITAL LETTER Z WITH DESCENDER +2C6D; C; 0251; # LATIN CAPITAL LETTER ALPHA +2C6E; C; 0271; # LATIN CAPITAL LETTER M WITH HOOK +2C6F; C; 0250; # LATIN CAPITAL LETTER TURNED A +2C70; C; 0252; # LATIN CAPITAL LETTER TURNED ALPHA +2C72; C; 2C73; # LATIN CAPITAL LETTER W WITH HOOK +2C75; C; 2C76; # LATIN CAPITAL LETTER HALF H +2C7E; C; 023F; # LATIN CAPITAL LETTER S WITH SWASH TAIL +2C7F; C; 0240; # LATIN CAPITAL LETTER Z WITH SWASH TAIL +2C80; C; 2C81; # COPTIC CAPITAL LETTER ALFA +2C82; C; 2C83; # COPTIC CAPITAL LETTER VIDA +2C84; C; 2C85; # COPTIC CAPITAL LETTER GAMMA +2C86; C; 2C87; # COPTIC CAPITAL LETTER DALDA +2C88; C; 2C89; # COPTIC CAPITAL LETTER EIE +2C8A; C; 2C8B; # COPTIC CAPITAL LETTER SOU +2C8C; C; 2C8D; # COPTIC CAPITAL LETTER ZATA +2C8E; C; 2C8F; # COPTIC CAPITAL LETTER HATE +2C90; C; 2C91; # COPTIC CAPITAL LETTER THETHE +2C92; C; 2C93; # COPTIC CAPITAL LETTER IAUDA +2C94; C; 2C95; # COPTIC CAPITAL LETTER KAPA +2C96; C; 2C97; # COPTIC CAPITAL LETTER LAULA +2C98; C; 2C99; # COPTIC CAPITAL LETTER MI +2C9A; C; 2C9B; # COPTIC CAPITAL LETTER NI +2C9C; C; 2C9D; # COPTIC CAPITAL LETTER KSI +2C9E; C; 2C9F; # COPTIC CAPITAL LETTER O +2CA0; C; 2CA1; # COPTIC CAPITAL LETTER PI +2CA2; C; 2CA3; # COPTIC CAPITAL LETTER RO +2CA4; C; 2CA5; # COPTIC CAPITAL LETTER SIMA +2CA6; C; 2CA7; # COPTIC CAPITAL LETTER TAU +2CA8; C; 2CA9; # COPTIC CAPITAL LETTER UA +2CAA; C; 2CAB; # COPTIC CAPITAL LETTER FI +2CAC; C; 2CAD; # COPTIC CAPITAL LETTER KHI +2CAE; C; 2CAF; # COPTIC CAPITAL LETTER PSI +2CB0; C; 2CB1; # COPTIC CAPITAL LETTER OOU +2CB2; C; 2CB3; # COPTIC CAPITAL LETTER DIALECT-P ALEF +2CB4; C; 2CB5; # COPTIC CAPITAL LETTER OLD COPTIC AIN +2CB6; C; 2CB7; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC EIE +2CB8; C; 2CB9; # COPTIC CAPITAL LETTER DIALECT-P KAPA +2CBA; C; 2CBB; # COPTIC CAPITAL LETTER DIALECT-P NI +2CBC; C; 2CBD; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC NI +2CBE; C; 2CBF; # COPTIC CAPITAL LETTER OLD COPTIC OOU +2CC0; C; 2CC1; # COPTIC CAPITAL LETTER SAMPI +2CC2; C; 2CC3; # COPTIC CAPITAL LETTER CROSSED SHEI +2CC4; C; 2CC5; # COPTIC CAPITAL LETTER OLD COPTIC SHEI +2CC6; C; 2CC7; # COPTIC CAPITAL LETTER OLD COPTIC ESH +2CC8; C; 2CC9; # COPTIC CAPITAL LETTER AKHMIMIC KHEI +2CCA; C; 2CCB; # COPTIC CAPITAL LETTER DIALECT-P HORI +2CCC; C; 2CCD; # COPTIC CAPITAL LETTER OLD COPTIC HORI +2CCE; C; 2CCF; # COPTIC CAPITAL LETTER OLD COPTIC HA +2CD0; C; 2CD1; # COPTIC CAPITAL LETTER L-SHAPED HA +2CD2; C; 2CD3; # COPTIC CAPITAL LETTER OLD COPTIC HEI +2CD4; C; 2CD5; # COPTIC CAPITAL LETTER OLD COPTIC HAT +2CD6; C; 2CD7; # COPTIC CAPITAL LETTER OLD COPTIC GANGIA +2CD8; C; 2CD9; # COPTIC CAPITAL LETTER OLD COPTIC DJA +2CDA; C; 2CDB; # COPTIC CAPITAL LETTER OLD COPTIC SHIMA +2CDC; C; 2CDD; # COPTIC CAPITAL LETTER OLD NUBIAN SHIMA +2CDE; C; 2CDF; # COPTIC CAPITAL LETTER OLD NUBIAN NGI +2CE0; C; 2CE1; # COPTIC CAPITAL LETTER OLD NUBIAN NYI +2CE2; C; 2CE3; # COPTIC CAPITAL LETTER OLD NUBIAN WAU +2CEB; C; 2CEC; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI +2CED; C; 2CEE; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC GANGIA +2CF2; C; 2CF3; # COPTIC CAPITAL LETTER BOHAIRIC KHEI +A640; C; A641; # CYRILLIC CAPITAL LETTER ZEMLYA +A642; C; A643; # CYRILLIC CAPITAL LETTER DZELO +A644; C; A645; # CYRILLIC CAPITAL LETTER REVERSED DZE +A646; C; A647; # CYRILLIC CAPITAL LETTER IOTA +A648; C; A649; # CYRILLIC CAPITAL LETTER DJERV +A64A; C; A64B; # CYRILLIC CAPITAL LETTER MONOGRAPH UK +A64C; C; A64D; # CYRILLIC CAPITAL LETTER BROAD OMEGA +A64E; C; A64F; # CYRILLIC CAPITAL LETTER NEUTRAL YER +A650; C; A651; # CYRILLIC CAPITAL LETTER YERU WITH BACK YER +A652; C; A653; # CYRILLIC CAPITAL LETTER IOTIFIED YAT +A654; C; A655; # CYRILLIC CAPITAL LETTER REVERSED YU +A656; C; A657; # CYRILLIC CAPITAL LETTER IOTIFIED A +A658; C; A659; # CYRILLIC CAPITAL LETTER CLOSED LITTLE YUS +A65A; C; A65B; # CYRILLIC CAPITAL LETTER BLENDED YUS +A65C; C; A65D; # CYRILLIC CAPITAL LETTER IOTIFIED CLOSED LITTLE YUS +A65E; C; A65F; # CYRILLIC CAPITAL LETTER YN +A660; C; A661; # CYRILLIC CAPITAL LETTER REVERSED TSE +A662; C; A663; # CYRILLIC CAPITAL LETTER SOFT DE +A664; C; A665; # CYRILLIC CAPITAL LETTER SOFT EL +A666; C; A667; # CYRILLIC CAPITAL LETTER SOFT EM +A668; C; A669; # CYRILLIC CAPITAL LETTER MONOCULAR O +A66A; C; A66B; # CYRILLIC CAPITAL LETTER BINOCULAR O +A66C; C; A66D; # CYRILLIC CAPITAL LETTER DOUBLE MONOCULAR O +A680; C; A681; # CYRILLIC CAPITAL LETTER DWE +A682; C; A683; # CYRILLIC CAPITAL LETTER DZWE +A684; C; A685; # CYRILLIC CAPITAL LETTER ZHWE +A686; C; A687; # CYRILLIC CAPITAL LETTER CCHE +A688; C; A689; # CYRILLIC CAPITAL LETTER DZZE +A68A; C; A68B; # CYRILLIC CAPITAL LETTER TE WITH MIDDLE HOOK +A68C; C; A68D; # CYRILLIC CAPITAL LETTER TWE +A68E; C; A68F; # CYRILLIC CAPITAL LETTER TSWE +A690; C; A691; # CYRILLIC CAPITAL LETTER TSSE +A692; C; A693; # CYRILLIC CAPITAL LETTER TCHE +A694; C; A695; # CYRILLIC CAPITAL LETTER HWE +A696; C; A697; # CYRILLIC CAPITAL LETTER SHWE +A698; C; A699; # CYRILLIC CAPITAL LETTER DOUBLE O +A69A; C; A69B; # CYRILLIC CAPITAL LETTER CROSSED O +A722; C; A723; # LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF +A724; C; A725; # LATIN CAPITAL LETTER EGYPTOLOGICAL AIN +A726; C; A727; # LATIN CAPITAL LETTER HENG +A728; C; A729; # LATIN CAPITAL LETTER TZ +A72A; C; A72B; # LATIN CAPITAL LETTER TRESILLO +A72C; C; A72D; # LATIN CAPITAL LETTER CUATRILLO +A72E; C; A72F; # LATIN CAPITAL LETTER CUATRILLO WITH COMMA +A732; C; A733; # LATIN CAPITAL LETTER AA +A734; C; A735; # LATIN CAPITAL LETTER AO +A736; C; A737; # LATIN CAPITAL LETTER AU +A738; C; A739; # LATIN CAPITAL LETTER AV +A73A; C; A73B; # LATIN CAPITAL LETTER AV WITH HORIZONTAL BAR +A73C; C; A73D; # LATIN CAPITAL LETTER AY +A73E; C; A73F; # LATIN CAPITAL LETTER REVERSED C WITH DOT +A740; C; A741; # LATIN CAPITAL LETTER K WITH STROKE +A742; C; A743; # LATIN CAPITAL LETTER K WITH DIAGONAL STROKE +A744; C; A745; # LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE +A746; C; A747; # LATIN CAPITAL LETTER BROKEN L +A748; C; A749; # LATIN CAPITAL LETTER L WITH HIGH STROKE +A74A; C; A74B; # LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY +A74C; C; A74D; # LATIN CAPITAL LETTER O WITH LOOP +A74E; C; A74F; # LATIN CAPITAL LETTER OO +A750; C; A751; # LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER +A752; C; A753; # LATIN CAPITAL LETTER P WITH FLOURISH +A754; C; A755; # LATIN CAPITAL LETTER P WITH SQUIRREL TAIL +A756; C; A757; # LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER +A758; C; A759; # LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE +A75A; C; A75B; # LATIN CAPITAL LETTER R ROTUNDA +A75C; C; A75D; # LATIN CAPITAL LETTER RUM ROTUNDA +A75E; C; A75F; # LATIN CAPITAL LETTER V WITH DIAGONAL STROKE +A760; C; A761; # LATIN CAPITAL LETTER VY +A762; C; A763; # LATIN CAPITAL LETTER VISIGOTHIC Z +A764; C; A765; # LATIN CAPITAL LETTER THORN WITH STROKE +A766; C; A767; # LATIN CAPITAL LETTER THORN WITH STROKE THROUGH DESCENDER +A768; C; A769; # LATIN CAPITAL LETTER VEND +A76A; C; A76B; # LATIN CAPITAL LETTER ET +A76C; C; A76D; # LATIN CAPITAL LETTER IS +A76E; C; A76F; # LATIN CAPITAL LETTER CON +A779; C; A77A; # LATIN CAPITAL LETTER INSULAR D +A77B; C; A77C; # LATIN CAPITAL LETTER INSULAR F +A77D; C; 1D79; # LATIN CAPITAL LETTER INSULAR G +A77E; C; A77F; # LATIN CAPITAL LETTER TURNED INSULAR G +A780; C; A781; # LATIN CAPITAL LETTER TURNED L +A782; C; A783; # LATIN CAPITAL LETTER INSULAR R +A784; C; A785; # LATIN CAPITAL LETTER INSULAR S +A786; C; A787; # LATIN CAPITAL LETTER INSULAR T +A78B; C; A78C; # LATIN CAPITAL LETTER SALTILLO +A78D; C; 0265; # LATIN CAPITAL LETTER TURNED H +A790; C; A791; # LATIN CAPITAL LETTER N WITH DESCENDER +A792; C; A793; # LATIN CAPITAL LETTER C WITH BAR +A796; C; A797; # LATIN CAPITAL LETTER B WITH FLOURISH +A798; C; A799; # LATIN CAPITAL LETTER F WITH STROKE +A79A; C; A79B; # LATIN CAPITAL LETTER VOLAPUK AE +A79C; C; A79D; # LATIN CAPITAL LETTER VOLAPUK OE +A79E; C; A79F; # LATIN CAPITAL LETTER VOLAPUK UE +A7A0; C; A7A1; # LATIN CAPITAL LETTER G WITH OBLIQUE STROKE +A7A2; C; A7A3; # LATIN CAPITAL LETTER K WITH OBLIQUE STROKE +A7A4; C; A7A5; # LATIN CAPITAL LETTER N WITH OBLIQUE STROKE +A7A6; C; A7A7; # LATIN CAPITAL LETTER R WITH OBLIQUE STROKE +A7A8; C; A7A9; # LATIN CAPITAL LETTER S WITH OBLIQUE STROKE +A7AA; C; 0266; # LATIN CAPITAL LETTER H WITH HOOK +A7AB; C; 025C; # LATIN CAPITAL LETTER REVERSED OPEN E +A7AC; C; 0261; # LATIN CAPITAL LETTER SCRIPT G +A7AD; C; 026C; # LATIN CAPITAL LETTER L WITH BELT +A7AE; C; 026A; # LATIN CAPITAL LETTER SMALL CAPITAL I +A7B0; C; 029E; # LATIN CAPITAL LETTER TURNED K +A7B1; C; 0287; # LATIN CAPITAL LETTER TURNED T +A7B2; C; 029D; # LATIN CAPITAL LETTER J WITH CROSSED-TAIL +A7B3; C; AB53; # LATIN CAPITAL LETTER CHI +A7B4; C; A7B5; # LATIN CAPITAL LETTER BETA +A7B6; C; A7B7; # LATIN CAPITAL LETTER OMEGA +AB70; C; 13A0; # CHEROKEE SMALL LETTER A +AB71; C; 13A1; # CHEROKEE SMALL LETTER E +AB72; C; 13A2; # CHEROKEE SMALL LETTER I +AB73; C; 13A3; # CHEROKEE SMALL LETTER O +AB74; C; 13A4; # CHEROKEE SMALL LETTER U +AB75; C; 13A5; # CHEROKEE SMALL LETTER V +AB76; C; 13A6; # CHEROKEE SMALL LETTER GA +AB77; C; 13A7; # CHEROKEE SMALL LETTER KA +AB78; C; 13A8; # CHEROKEE SMALL LETTER GE +AB79; C; 13A9; # CHEROKEE SMALL LETTER GI +AB7A; C; 13AA; # CHEROKEE SMALL LETTER GO +AB7B; C; 13AB; # CHEROKEE SMALL LETTER GU +AB7C; C; 13AC; # CHEROKEE SMALL LETTER GV +AB7D; C; 13AD; # CHEROKEE SMALL LETTER HA +AB7E; C; 13AE; # CHEROKEE SMALL LETTER HE +AB7F; C; 13AF; # CHEROKEE SMALL LETTER HI +AB80; C; 13B0; # CHEROKEE SMALL LETTER HO +AB81; C; 13B1; # CHEROKEE SMALL LETTER HU +AB82; C; 13B2; # CHEROKEE SMALL LETTER HV +AB83; C; 13B3; # CHEROKEE SMALL LETTER LA +AB84; C; 13B4; # CHEROKEE SMALL LETTER LE +AB85; C; 13B5; # CHEROKEE SMALL LETTER LI +AB86; C; 13B6; # CHEROKEE SMALL LETTER LO +AB87; C; 13B7; # CHEROKEE SMALL LETTER LU +AB88; C; 13B8; # CHEROKEE SMALL LETTER LV +AB89; C; 13B9; # CHEROKEE SMALL LETTER MA +AB8A; C; 13BA; # CHEROKEE SMALL LETTER ME +AB8B; C; 13BB; # CHEROKEE SMALL LETTER MI +AB8C; C; 13BC; # CHEROKEE SMALL LETTER MO +AB8D; C; 13BD; # CHEROKEE SMALL LETTER MU +AB8E; C; 13BE; # CHEROKEE SMALL LETTER NA +AB8F; C; 13BF; # CHEROKEE SMALL LETTER HNA +AB90; C; 13C0; # CHEROKEE SMALL LETTER NAH +AB91; C; 13C1; # CHEROKEE SMALL LETTER NE +AB92; C; 13C2; # CHEROKEE SMALL LETTER NI +AB93; C; 13C3; # CHEROKEE SMALL LETTER NO +AB94; C; 13C4; # CHEROKEE SMALL LETTER NU +AB95; C; 13C5; # CHEROKEE SMALL LETTER NV +AB96; C; 13C6; # CHEROKEE SMALL LETTER QUA +AB97; C; 13C7; # CHEROKEE SMALL LETTER QUE +AB98; C; 13C8; # CHEROKEE SMALL LETTER QUI +AB99; C; 13C9; # CHEROKEE SMALL LETTER QUO +AB9A; C; 13CA; # CHEROKEE SMALL LETTER QUU +AB9B; C; 13CB; # CHEROKEE SMALL LETTER QUV +AB9C; C; 13CC; # CHEROKEE SMALL LETTER SA +AB9D; C; 13CD; # CHEROKEE SMALL LETTER S +AB9E; C; 13CE; # CHEROKEE SMALL LETTER SE +AB9F; C; 13CF; # CHEROKEE SMALL LETTER SI +ABA0; C; 13D0; # CHEROKEE SMALL LETTER SO +ABA1; C; 13D1; # CHEROKEE SMALL LETTER SU +ABA2; C; 13D2; # CHEROKEE SMALL LETTER SV +ABA3; C; 13D3; # CHEROKEE SMALL LETTER DA +ABA4; C; 13D4; # CHEROKEE SMALL LETTER TA +ABA5; C; 13D5; # CHEROKEE SMALL LETTER DE +ABA6; C; 13D6; # CHEROKEE SMALL LETTER TE +ABA7; C; 13D7; # CHEROKEE SMALL LETTER DI +ABA8; C; 13D8; # CHEROKEE SMALL LETTER TI +ABA9; C; 13D9; # CHEROKEE SMALL LETTER DO +ABAA; C; 13DA; # CHEROKEE SMALL LETTER DU +ABAB; C; 13DB; # CHEROKEE SMALL LETTER DV +ABAC; C; 13DC; # CHEROKEE SMALL LETTER DLA +ABAD; C; 13DD; # CHEROKEE SMALL LETTER TLA +ABAE; C; 13DE; # CHEROKEE SMALL LETTER TLE +ABAF; C; 13DF; # CHEROKEE SMALL LETTER TLI +ABB0; C; 13E0; # CHEROKEE SMALL LETTER TLO +ABB1; C; 13E1; # CHEROKEE SMALL LETTER TLU +ABB2; C; 13E2; # CHEROKEE SMALL LETTER TLV +ABB3; C; 13E3; # CHEROKEE SMALL LETTER TSA +ABB4; C; 13E4; # CHEROKEE SMALL LETTER TSE +ABB5; C; 13E5; # CHEROKEE SMALL LETTER TSI +ABB6; C; 13E6; # CHEROKEE SMALL LETTER TSO +ABB7; C; 13E7; # CHEROKEE SMALL LETTER TSU +ABB8; C; 13E8; # CHEROKEE SMALL LETTER TSV +ABB9; C; 13E9; # CHEROKEE SMALL LETTER WA +ABBA; C; 13EA; # CHEROKEE SMALL LETTER WE +ABBB; C; 13EB; # CHEROKEE SMALL LETTER WI +ABBC; C; 13EC; # CHEROKEE SMALL LETTER WO +ABBD; C; 13ED; # CHEROKEE SMALL LETTER WU +ABBE; C; 13EE; # CHEROKEE SMALL LETTER WV +ABBF; C; 13EF; # CHEROKEE SMALL LETTER YA +FB00; F; 0066 0066; # LATIN SMALL LIGATURE FF +FB01; F; 0066 0069; # LATIN SMALL LIGATURE FI +FB02; F; 0066 006C; # LATIN SMALL LIGATURE FL +FB03; F; 0066 0066 0069; # LATIN SMALL LIGATURE FFI +FB04; F; 0066 0066 006C; # LATIN SMALL LIGATURE FFL +FB05; F; 0073 0074; # LATIN SMALL LIGATURE LONG S T +FB06; F; 0073 0074; # LATIN SMALL LIGATURE ST +FB13; F; 0574 0576; # ARMENIAN SMALL LIGATURE MEN NOW +FB14; F; 0574 0565; # ARMENIAN SMALL LIGATURE MEN ECH +FB15; F; 0574 056B; # ARMENIAN SMALL LIGATURE MEN INI +FB16; F; 057E 0576; # ARMENIAN SMALL LIGATURE VEW NOW +FB17; F; 0574 056D; # ARMENIAN SMALL LIGATURE MEN XEH +FF21; C; FF41; # FULLWIDTH LATIN CAPITAL LETTER A +FF22; C; FF42; # FULLWIDTH LATIN CAPITAL LETTER B +FF23; C; FF43; # FULLWIDTH LATIN CAPITAL LETTER C +FF24; C; FF44; # FULLWIDTH LATIN CAPITAL LETTER D +FF25; C; FF45; # FULLWIDTH LATIN CAPITAL LETTER E +FF26; C; FF46; # FULLWIDTH LATIN CAPITAL LETTER F +FF27; C; FF47; # FULLWIDTH LATIN CAPITAL LETTER G +FF28; C; FF48; # FULLWIDTH LATIN CAPITAL LETTER H +FF29; C; FF49; # FULLWIDTH LATIN CAPITAL LETTER I +FF2A; C; FF4A; # FULLWIDTH LATIN CAPITAL LETTER J +FF2B; C; FF4B; # FULLWIDTH LATIN CAPITAL LETTER K +FF2C; C; FF4C; # FULLWIDTH LATIN CAPITAL LETTER L +FF2D; C; FF4D; # FULLWIDTH LATIN CAPITAL LETTER M +FF2E; C; FF4E; # FULLWIDTH LATIN CAPITAL LETTER N +FF2F; C; FF4F; # FULLWIDTH LATIN CAPITAL LETTER O +FF30; C; FF50; # FULLWIDTH LATIN CAPITAL LETTER P +FF31; C; FF51; # FULLWIDTH LATIN CAPITAL LETTER Q +FF32; C; FF52; # FULLWIDTH LATIN CAPITAL LETTER R +FF33; C; FF53; # FULLWIDTH LATIN CAPITAL LETTER S +FF34; C; FF54; # FULLWIDTH LATIN CAPITAL LETTER T +FF35; C; FF55; # FULLWIDTH LATIN CAPITAL LETTER U +FF36; C; FF56; # FULLWIDTH LATIN CAPITAL LETTER V +FF37; C; FF57; # FULLWIDTH LATIN CAPITAL LETTER W +FF38; C; FF58; # FULLWIDTH LATIN CAPITAL LETTER X +FF39; C; FF59; # FULLWIDTH LATIN CAPITAL LETTER Y +FF3A; C; FF5A; # FULLWIDTH LATIN CAPITAL LETTER Z +10400; C; 10428; # DESERET CAPITAL LETTER LONG I +10401; C; 10429; # DESERET CAPITAL LETTER LONG E +10402; C; 1042A; # DESERET CAPITAL LETTER LONG A +10403; C; 1042B; # DESERET CAPITAL LETTER LONG AH +10404; C; 1042C; # DESERET CAPITAL LETTER LONG O +10405; C; 1042D; # DESERET CAPITAL LETTER LONG OO +10406; C; 1042E; # DESERET CAPITAL LETTER SHORT I +10407; C; 1042F; # DESERET CAPITAL LETTER SHORT E +10408; C; 10430; # DESERET CAPITAL LETTER SHORT A +10409; C; 10431; # DESERET CAPITAL LETTER SHORT AH +1040A; C; 10432; # DESERET CAPITAL LETTER SHORT O +1040B; C; 10433; # DESERET CAPITAL LETTER SHORT OO +1040C; C; 10434; # DESERET CAPITAL LETTER AY +1040D; C; 10435; # DESERET CAPITAL LETTER OW +1040E; C; 10436; # DESERET CAPITAL LETTER WU +1040F; C; 10437; # DESERET CAPITAL LETTER YEE +10410; C; 10438; # DESERET CAPITAL LETTER H +10411; C; 10439; # DESERET CAPITAL LETTER PEE +10412; C; 1043A; # DESERET CAPITAL LETTER BEE +10413; C; 1043B; # DESERET CAPITAL LETTER TEE +10414; C; 1043C; # DESERET CAPITAL LETTER DEE +10415; C; 1043D; # DESERET CAPITAL LETTER CHEE +10416; C; 1043E; # DESERET CAPITAL LETTER JEE +10417; C; 1043F; # DESERET CAPITAL LETTER KAY +10418; C; 10440; # DESERET CAPITAL LETTER GAY +10419; C; 10441; # DESERET CAPITAL LETTER EF +1041A; C; 10442; # DESERET CAPITAL LETTER VEE +1041B; C; 10443; # DESERET CAPITAL LETTER ETH +1041C; C; 10444; # DESERET CAPITAL LETTER THEE +1041D; C; 10445; # DESERET CAPITAL LETTER ES +1041E; C; 10446; # DESERET CAPITAL LETTER ZEE +1041F; C; 10447; # DESERET CAPITAL LETTER ESH +10420; C; 10448; # DESERET CAPITAL LETTER ZHEE +10421; C; 10449; # DESERET CAPITAL LETTER ER +10422; C; 1044A; # DESERET CAPITAL LETTER EL +10423; C; 1044B; # DESERET CAPITAL LETTER EM +10424; C; 1044C; # DESERET CAPITAL LETTER EN +10425; C; 1044D; # DESERET CAPITAL LETTER ENG +10426; C; 1044E; # DESERET CAPITAL LETTER OI +10427; C; 1044F; # DESERET CAPITAL LETTER EW +104B0; C; 104D8; # OSAGE CAPITAL LETTER A +104B1; C; 104D9; # OSAGE CAPITAL LETTER AI +104B2; C; 104DA; # OSAGE CAPITAL LETTER AIN +104B3; C; 104DB; # OSAGE CAPITAL LETTER AH +104B4; C; 104DC; # OSAGE CAPITAL LETTER BRA +104B5; C; 104DD; # OSAGE CAPITAL LETTER CHA +104B6; C; 104DE; # OSAGE CAPITAL LETTER EHCHA +104B7; C; 104DF; # OSAGE CAPITAL LETTER E +104B8; C; 104E0; # OSAGE CAPITAL LETTER EIN +104B9; C; 104E1; # OSAGE CAPITAL LETTER HA +104BA; C; 104E2; # OSAGE CAPITAL LETTER HYA +104BB; C; 104E3; # OSAGE CAPITAL LETTER I +104BC; C; 104E4; # OSAGE CAPITAL LETTER KA +104BD; C; 104E5; # OSAGE CAPITAL LETTER EHKA +104BE; C; 104E6; # OSAGE CAPITAL LETTER KYA +104BF; C; 104E7; # OSAGE CAPITAL LETTER LA +104C0; C; 104E8; # OSAGE CAPITAL LETTER MA +104C1; C; 104E9; # OSAGE CAPITAL LETTER NA +104C2; C; 104EA; # OSAGE CAPITAL LETTER O +104C3; C; 104EB; # OSAGE CAPITAL LETTER OIN +104C4; C; 104EC; # OSAGE CAPITAL LETTER PA +104C5; C; 104ED; # OSAGE CAPITAL LETTER EHPA +104C6; C; 104EE; # OSAGE CAPITAL LETTER SA +104C7; C; 104EF; # OSAGE CAPITAL LETTER SHA +104C8; C; 104F0; # OSAGE CAPITAL LETTER TA +104C9; C; 104F1; # OSAGE CAPITAL LETTER EHTA +104CA; C; 104F2; # OSAGE CAPITAL LETTER TSA +104CB; C; 104F3; # OSAGE CAPITAL LETTER EHTSA +104CC; C; 104F4; # OSAGE CAPITAL LETTER TSHA +104CD; C; 104F5; # OSAGE CAPITAL LETTER DHA +104CE; C; 104F6; # OSAGE CAPITAL LETTER U +104CF; C; 104F7; # OSAGE CAPITAL LETTER WA +104D0; C; 104F8; # OSAGE CAPITAL LETTER KHA +104D1; C; 104F9; # OSAGE CAPITAL LETTER GHA +104D2; C; 104FA; # OSAGE CAPITAL LETTER ZA +104D3; C; 104FB; # OSAGE CAPITAL LETTER ZHA +10C80; C; 10CC0; # OLD HUNGARIAN CAPITAL LETTER A +10C81; C; 10CC1; # OLD HUNGARIAN CAPITAL LETTER AA +10C82; C; 10CC2; # OLD HUNGARIAN CAPITAL LETTER EB +10C83; C; 10CC3; # OLD HUNGARIAN CAPITAL LETTER AMB +10C84; C; 10CC4; # OLD HUNGARIAN CAPITAL LETTER EC +10C85; C; 10CC5; # OLD HUNGARIAN CAPITAL LETTER ENC +10C86; C; 10CC6; # OLD HUNGARIAN CAPITAL LETTER ECS +10C87; C; 10CC7; # OLD HUNGARIAN CAPITAL LETTER ED +10C88; C; 10CC8; # OLD HUNGARIAN CAPITAL LETTER AND +10C89; C; 10CC9; # OLD HUNGARIAN CAPITAL LETTER E +10C8A; C; 10CCA; # OLD HUNGARIAN CAPITAL LETTER CLOSE E +10C8B; C; 10CCB; # OLD HUNGARIAN CAPITAL LETTER EE +10C8C; C; 10CCC; # OLD HUNGARIAN CAPITAL LETTER EF +10C8D; C; 10CCD; # OLD HUNGARIAN CAPITAL LETTER EG +10C8E; C; 10CCE; # OLD HUNGARIAN CAPITAL LETTER EGY +10C8F; C; 10CCF; # OLD HUNGARIAN CAPITAL LETTER EH +10C90; C; 10CD0; # OLD HUNGARIAN CAPITAL LETTER I +10C91; C; 10CD1; # OLD HUNGARIAN CAPITAL LETTER II +10C92; C; 10CD2; # OLD HUNGARIAN CAPITAL LETTER EJ +10C93; C; 10CD3; # OLD HUNGARIAN CAPITAL LETTER EK +10C94; C; 10CD4; # OLD HUNGARIAN CAPITAL LETTER AK +10C95; C; 10CD5; # OLD HUNGARIAN CAPITAL LETTER UNK +10C96; C; 10CD6; # OLD HUNGARIAN CAPITAL LETTER EL +10C97; C; 10CD7; # OLD HUNGARIAN CAPITAL LETTER ELY +10C98; C; 10CD8; # OLD HUNGARIAN CAPITAL LETTER EM +10C99; C; 10CD9; # OLD HUNGARIAN CAPITAL LETTER EN +10C9A; C; 10CDA; # OLD HUNGARIAN CAPITAL LETTER ENY +10C9B; C; 10CDB; # OLD HUNGARIAN CAPITAL LETTER O +10C9C; C; 10CDC; # OLD HUNGARIAN CAPITAL LETTER OO +10C9D; C; 10CDD; # OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG OE +10C9E; C; 10CDE; # OLD HUNGARIAN CAPITAL LETTER RUDIMENTA OE +10C9F; C; 10CDF; # OLD HUNGARIAN CAPITAL LETTER OEE +10CA0; C; 10CE0; # OLD HUNGARIAN CAPITAL LETTER EP +10CA1; C; 10CE1; # OLD HUNGARIAN CAPITAL LETTER EMP +10CA2; C; 10CE2; # OLD HUNGARIAN CAPITAL LETTER ER +10CA3; C; 10CE3; # OLD HUNGARIAN CAPITAL LETTER SHORT ER +10CA4; C; 10CE4; # OLD HUNGARIAN CAPITAL LETTER ES +10CA5; C; 10CE5; # OLD HUNGARIAN CAPITAL LETTER ESZ +10CA6; C; 10CE6; # OLD HUNGARIAN CAPITAL LETTER ET +10CA7; C; 10CE7; # OLD HUNGARIAN CAPITAL LETTER ENT +10CA8; C; 10CE8; # OLD HUNGARIAN CAPITAL LETTER ETY +10CA9; C; 10CE9; # OLD HUNGARIAN CAPITAL LETTER ECH +10CAA; C; 10CEA; # OLD HUNGARIAN CAPITAL LETTER U +10CAB; C; 10CEB; # OLD HUNGARIAN CAPITAL LETTER UU +10CAC; C; 10CEC; # OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG UE +10CAD; C; 10CED; # OLD HUNGARIAN CAPITAL LETTER RUDIMENTA UE +10CAE; C; 10CEE; # OLD HUNGARIAN CAPITAL LETTER EV +10CAF; C; 10CEF; # OLD HUNGARIAN CAPITAL LETTER EZ +10CB0; C; 10CF0; # OLD HUNGARIAN CAPITAL LETTER EZS +10CB1; C; 10CF1; # OLD HUNGARIAN CAPITAL LETTER ENT-SHAPED SIGN +10CB2; C; 10CF2; # OLD HUNGARIAN CAPITAL LETTER US +118A0; C; 118C0; # WARANG CITI CAPITAL LETTER NGAA +118A1; C; 118C1; # WARANG CITI CAPITAL LETTER A +118A2; C; 118C2; # WARANG CITI CAPITAL LETTER WI +118A3; C; 118C3; # WARANG CITI CAPITAL LETTER YU +118A4; C; 118C4; # WARANG CITI CAPITAL LETTER YA +118A5; C; 118C5; # WARANG CITI CAPITAL LETTER YO +118A6; C; 118C6; # WARANG CITI CAPITAL LETTER II +118A7; C; 118C7; # WARANG CITI CAPITAL LETTER UU +118A8; C; 118C8; # WARANG CITI CAPITAL LETTER E +118A9; C; 118C9; # WARANG CITI CAPITAL LETTER O +118AA; C; 118CA; # WARANG CITI CAPITAL LETTER ANG +118AB; C; 118CB; # WARANG CITI CAPITAL LETTER GA +118AC; C; 118CC; # WARANG CITI CAPITAL LETTER KO +118AD; C; 118CD; # WARANG CITI CAPITAL LETTER ENY +118AE; C; 118CE; # WARANG CITI CAPITAL LETTER YUJ +118AF; C; 118CF; # WARANG CITI CAPITAL LETTER UC +118B0; C; 118D0; # WARANG CITI CAPITAL LETTER ENN +118B1; C; 118D1; # WARANG CITI CAPITAL LETTER ODD +118B2; C; 118D2; # WARANG CITI CAPITAL LETTER TTE +118B3; C; 118D3; # WARANG CITI CAPITAL LETTER NUNG +118B4; C; 118D4; # WARANG CITI CAPITAL LETTER DA +118B5; C; 118D5; # WARANG CITI CAPITAL LETTER AT +118B6; C; 118D6; # WARANG CITI CAPITAL LETTER AM +118B7; C; 118D7; # WARANG CITI CAPITAL LETTER BU +118B8; C; 118D8; # WARANG CITI CAPITAL LETTER PU +118B9; C; 118D9; # WARANG CITI CAPITAL LETTER HIYO +118BA; C; 118DA; # WARANG CITI CAPITAL LETTER HOLO +118BB; C; 118DB; # WARANG CITI CAPITAL LETTER HORR +118BC; C; 118DC; # WARANG CITI CAPITAL LETTER HAR +118BD; C; 118DD; # WARANG CITI CAPITAL LETTER SSUU +118BE; C; 118DE; # WARANG CITI CAPITAL LETTER SII +118BF; C; 118DF; # WARANG CITI CAPITAL LETTER VIYO +1E900; C; 1E922; # ADLAM CAPITAL LETTER ALIF +1E901; C; 1E923; # ADLAM CAPITAL LETTER DAALI +1E902; C; 1E924; # ADLAM CAPITAL LETTER LAAM +1E903; C; 1E925; # ADLAM CAPITAL LETTER MIIM +1E904; C; 1E926; # ADLAM CAPITAL LETTER BA +1E905; C; 1E927; # ADLAM CAPITAL LETTER SINNYIIYHE +1E906; C; 1E928; # ADLAM CAPITAL LETTER PE +1E907; C; 1E929; # ADLAM CAPITAL LETTER BHE +1E908; C; 1E92A; # ADLAM CAPITAL LETTER RA +1E909; C; 1E92B; # ADLAM CAPITAL LETTER E +1E90A; C; 1E92C; # ADLAM CAPITAL LETTER FA +1E90B; C; 1E92D; # ADLAM CAPITAL LETTER I +1E90C; C; 1E92E; # ADLAM CAPITAL LETTER O +1E90D; C; 1E92F; # ADLAM CAPITAL LETTER DHA +1E90E; C; 1E930; # ADLAM CAPITAL LETTER YHE +1E90F; C; 1E931; # ADLAM CAPITAL LETTER WAW +1E910; C; 1E932; # ADLAM CAPITAL LETTER NUN +1E911; C; 1E933; # ADLAM CAPITAL LETTER KAF +1E912; C; 1E934; # ADLAM CAPITAL LETTER YA +1E913; C; 1E935; # ADLAM CAPITAL LETTER U +1E914; C; 1E936; # ADLAM CAPITAL LETTER JIIM +1E915; C; 1E937; # ADLAM CAPITAL LETTER CHI +1E916; C; 1E938; # ADLAM CAPITAL LETTER HA +1E917; C; 1E939; # ADLAM CAPITAL LETTER QAAF +1E918; C; 1E93A; # ADLAM CAPITAL LETTER GA +1E919; C; 1E93B; # ADLAM CAPITAL LETTER NYA +1E91A; C; 1E93C; # ADLAM CAPITAL LETTER TU +1E91B; C; 1E93D; # ADLAM CAPITAL LETTER NHA +1E91C; C; 1E93E; # ADLAM CAPITAL LETTER VA +1E91D; C; 1E93F; # ADLAM CAPITAL LETTER KHA +1E91E; C; 1E940; # ADLAM CAPITAL LETTER GBE +1E91F; C; 1E941; # ADLAM CAPITAL LETTER ZAL +1E920; C; 1E942; # ADLAM CAPITAL LETTER KPO +1E921; C; 1E943; # ADLAM CAPITAL LETTER SHA +# +# EOF diff --git a/rpython/rlib/unicodedata/CompositionExclusions-9.0.0.txt b/rpython/rlib/unicodedata/CompositionExclusions-9.0.0.txt new file mode 100644 --- /dev/null +++ b/rpython/rlib/unicodedata/CompositionExclusions-9.0.0.txt @@ -0,0 +1,208 @@ +# CompositionExclusions-9.0.0.txt +# Date: 2016-01-21, 22:00:00 GMT [KW, LI] +# © 2016 Unicode®, Inc. +# For terms of use, see http://www.unicode.org/terms_of_use.html +# +# Unicode Character Database +# For documentation, see http://www.unicode.org/reports/tr44/ +# +# This file lists the characters for the Composition Exclusion Table +# defined in UAX #15, Unicode Normalization Forms. +# +# This file is a normative contributory data file in the +# Unicode Character Database. +# +# For more information, see +# http://www.unicode.org/unicode/reports/tr15/#Primary_Exclusion_List_Table +# +# For a full derivation of composition exclusions, see the derived property +# Full_Composition_Exclusion in DerivedNormalizationProps.txt +# + +# ================================================ +# (1) Script Specifics +# +# This list of characters cannot be derived from the UnicodeData.txt file. +# ================================================ + +0958 # DEVANAGARI LETTER QA +0959 # DEVANAGARI LETTER KHHA +095A # DEVANAGARI LETTER GHHA +095B # DEVANAGARI LETTER ZA +095C # DEVANAGARI LETTER DDDHA +095D # DEVANAGARI LETTER RHA +095E # DEVANAGARI LETTER FA +095F # DEVANAGARI LETTER YYA +09DC # BENGALI LETTER RRA +09DD # BENGALI LETTER RHA +09DF # BENGALI LETTER YYA +0A33 # GURMUKHI LETTER LLA +0A36 # GURMUKHI LETTER SHA +0A59 # GURMUKHI LETTER KHHA +0A5A # GURMUKHI LETTER GHHA +0A5B # GURMUKHI LETTER ZA +0A5E # GURMUKHI LETTER FA +0B5C # ORIYA LETTER RRA +0B5D # ORIYA LETTER RHA +0F43 # TIBETAN LETTER GHA +0F4D # TIBETAN LETTER DDHA +0F52 # TIBETAN LETTER DHA +0F57 # TIBETAN LETTER BHA +0F5C # TIBETAN LETTER DZHA +0F69 # TIBETAN LETTER KSSA +0F76 # TIBETAN VOWEL SIGN VOCALIC R +0F78 # TIBETAN VOWEL SIGN VOCALIC L +0F93 # TIBETAN SUBJOINED LETTER GHA +0F9D # TIBETAN SUBJOINED LETTER DDHA +0FA2 # TIBETAN SUBJOINED LETTER DHA +0FA7 # TIBETAN SUBJOINED LETTER BHA +0FAC # TIBETAN SUBJOINED LETTER DZHA +0FB9 # TIBETAN SUBJOINED LETTER KSSA +FB1D # HEBREW LETTER YOD WITH HIRIQ +FB1F # HEBREW LIGATURE YIDDISH YOD YOD PATAH +FB2A # HEBREW LETTER SHIN WITH SHIN DOT +FB2B # HEBREW LETTER SHIN WITH SIN DOT +FB2C # HEBREW LETTER SHIN WITH DAGESH AND SHIN DOT +FB2D # HEBREW LETTER SHIN WITH DAGESH AND SIN DOT +FB2E # HEBREW LETTER ALEF WITH PATAH +FB2F # HEBREW LETTER ALEF WITH QAMATS +FB30 # HEBREW LETTER ALEF WITH MAPIQ +FB31 # HEBREW LETTER BET WITH DAGESH +FB32 # HEBREW LETTER GIMEL WITH DAGESH +FB33 # HEBREW LETTER DALET WITH DAGESH +FB34 # HEBREW LETTER HE WITH MAPIQ +FB35 # HEBREW LETTER VAV WITH DAGESH +FB36 # HEBREW LETTER ZAYIN WITH DAGESH +FB38 # HEBREW LETTER TET WITH DAGESH +FB39 # HEBREW LETTER YOD WITH DAGESH +FB3A # HEBREW LETTER FINAL KAF WITH DAGESH +FB3B # HEBREW LETTER KAF WITH DAGESH +FB3C # HEBREW LETTER LAMED WITH DAGESH +FB3E # HEBREW LETTER MEM WITH DAGESH +FB40 # HEBREW LETTER NUN WITH DAGESH +FB41 # HEBREW LETTER SAMEKH WITH DAGESH +FB43 # HEBREW LETTER FINAL PE WITH DAGESH +FB44 # HEBREW LETTER PE WITH DAGESH +FB46 # HEBREW LETTER TSADI WITH DAGESH +FB47 # HEBREW LETTER QOF WITH DAGESH +FB48 # HEBREW LETTER RESH WITH DAGESH +FB49 # HEBREW LETTER SHIN WITH DAGESH +FB4A # HEBREW LETTER TAV WITH DAGESH +FB4B # HEBREW LETTER VAV WITH HOLAM +FB4C # HEBREW LETTER BET WITH RAFE +FB4D # HEBREW LETTER KAF WITH RAFE +FB4E # HEBREW LETTER PE WITH RAFE + +# Total code points: 67 + +# ================================================ +# (2) Post Composition Version precomposed characters +# +# These characters cannot be derived solely from the UnicodeData.txt file +# in this version of Unicode. +# +# Note that characters added to the standard after the +# Composition Version and which have canonical decomposition mappings +# are not automatically added to this list of Post Composition +# Version precomposed characters. +# ================================================ + +2ADC # FORKING +1D15E # MUSICAL SYMBOL HALF NOTE +1D15F # MUSICAL SYMBOL QUARTER NOTE +1D160 # MUSICAL SYMBOL EIGHTH NOTE +1D161 # MUSICAL SYMBOL SIXTEENTH NOTE +1D162 # MUSICAL SYMBOL THIRTY-SECOND NOTE +1D163 # MUSICAL SYMBOL SIXTY-FOURTH NOTE +1D164 # MUSICAL SYMBOL ONE HUNDRED TWENTY-EIGHTH NOTE +1D1BB # MUSICAL SYMBOL MINIMA +1D1BC # MUSICAL SYMBOL MINIMA BLACK +1D1BD # MUSICAL SYMBOL SEMIMINIMA WHITE +1D1BE # MUSICAL SYMBOL SEMIMINIMA BLACK +1D1BF # MUSICAL SYMBOL FUSA WHITE +1D1C0 # MUSICAL SYMBOL FUSA BLACK + +# Total code points: 14 + +# ================================================ +# (3) Singleton Decompositions +# +# These characters can be derived from the UnicodeData.txt file +# by including all canonically decomposable characters whose +# canonical decomposition consists of a single character. +# +# These characters are simply quoted here for reference. +# See also Full_Composition_Exclusion in DerivedNormalizationProps.txt +# ================================================ + +# 0340..0341 [2] COMBINING GRAVE TONE MARK..COMBINING ACUTE TONE MARK +# 0343 COMBINING GREEK KORONIS +# 0374 GREEK NUMERAL SIGN +# 037E GREEK QUESTION MARK +# 0387 GREEK ANO TELEIA +# 1F71 GREEK SMALL LETTER ALPHA WITH OXIA +# 1F73 GREEK SMALL LETTER EPSILON WITH OXIA +# 1F75 GREEK SMALL LETTER ETA WITH OXIA +# 1F77 GREEK SMALL LETTER IOTA WITH OXIA +# 1F79 GREEK SMALL LETTER OMICRON WITH OXIA +# 1F7B GREEK SMALL LETTER UPSILON WITH OXIA +# 1F7D GREEK SMALL LETTER OMEGA WITH OXIA +# 1FBB GREEK CAPITAL LETTER ALPHA WITH OXIA +# 1FBE GREEK PROSGEGRAMMENI +# 1FC9 GREEK CAPITAL LETTER EPSILON WITH OXIA +# 1FCB GREEK CAPITAL LETTER ETA WITH OXIA +# 1FD3 GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +# 1FDB GREEK CAPITAL LETTER IOTA WITH OXIA +# 1FE3 GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA +# 1FEB GREEK CAPITAL LETTER UPSILON WITH OXIA +# 1FEE..1FEF [2] GREEK DIALYTIKA AND OXIA..GREEK VARIA +# 1FF9 GREEK CAPITAL LETTER OMICRON WITH OXIA +# 1FFB GREEK CAPITAL LETTER OMEGA WITH OXIA +# 1FFD GREEK OXIA +# 2000..2001 [2] EN QUAD..EM QUAD +# 2126 OHM SIGN +# 212A..212B [2] KELVIN SIGN..ANGSTROM SIGN +# 2329 LEFT-POINTING ANGLE BRACKET +# 232A RIGHT-POINTING ANGLE BRACKET +# F900..FA0D [270] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA0D +# FA10 CJK COMPATIBILITY IDEOGRAPH-FA10 +# FA12 CJK COMPATIBILITY IDEOGRAPH-FA12 +# FA15..FA1E [10] CJK COMPATIBILITY IDEOGRAPH-FA15..CJK COMPATIBILITY IDEOGRAPH-FA1E +# FA20 CJK COMPATIBILITY IDEOGRAPH-FA20 +# FA22 CJK COMPATIBILITY IDEOGRAPH-FA22 +# FA25..FA26 [2] CJK COMPATIBILITY IDEOGRAPH-FA25..CJK COMPATIBILITY IDEOGRAPH-FA26 +# FA2A..FA6D [68] CJK COMPATIBILITY IDEOGRAPH-FA2A..CJK COMPATIBILITY IDEOGRAPH-FA6D +# FA70..FAD9 [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COMPATIBILITY IDEOGRAPH-FAD9 +# 2F800..2FA1D [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D + +# Total code points: 1035 + +# ================================================ +# (4) Non-Starter Decompositions +# +# These characters can be derived from the UnicodeData.txt file +# by including each expanding canonical decomposition +# (i.e., those which canonically decompose to a sequence +# of characters instead of a single character), such that: +# +# A. The character is not a Starter. +# +# OR (inclusive) +# +# B. The character's canonical decomposition begins +# with a character that is not a Starter. +# +# Note that a "Starter" is any character with a zero combining class. +# +# These characters are simply quoted here for reference. +# See also Full_Composition_Exclusion in DerivedNormalizationProps.txt +# ================================================ + +# 0344 COMBINING GREEK DIALYTIKA TONOS +# 0F73 TIBETAN VOWEL SIGN II +# 0F75 TIBETAN VOWEL SIGN UU +# 0F81 TIBETAN VOWEL SIGN REVERSED II + +# Total code points: 4 + +# EOF diff --git a/rpython/rlib/unicodedata/DerivedCoreProperties-9.0.0.txt b/rpython/rlib/unicodedata/DerivedCoreProperties-9.0.0.txt new file mode 100644 --- /dev/null +++ b/rpython/rlib/unicodedata/DerivedCoreProperties-9.0.0.txt @@ -0,0 +1,11309 @@ +# DerivedCoreProperties-9.0.0.txt +# Date: 2016-06-01, 10:34:24 GMT +# © 2016 Unicode®, Inc. +# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. +# For terms of use, see http://www.unicode.org/terms_of_use.html +# +# Unicode Character Database +# For documentation, see http://www.unicode.org/reports/tr44/ + +# ================================================ + +# Derived Property: Math +# Generated from: Sm + Other_Math + +002B ; Math # Sm PLUS SIGN +003C..003E ; Math # Sm [3] LESS-THAN SIGN..GREATER-THAN SIGN +005E ; Math # Sk CIRCUMFLEX ACCENT +007C ; Math # Sm VERTICAL LINE +007E ; Math # Sm TILDE +00AC ; Math # Sm NOT SIGN +00B1 ; Math # Sm PLUS-MINUS SIGN +00D7 ; Math # Sm MULTIPLICATION SIGN +00F7 ; Math # Sm DIVISION SIGN +03D0..03D2 ; Math # L& [3] GREEK BETA SYMBOL..GREEK UPSILON WITH HOOK SYMBOL +03D5 ; Math # L& GREEK PHI SYMBOL +03F0..03F1 ; Math # L& [2] GREEK KAPPA SYMBOL..GREEK RHO SYMBOL +03F4..03F5 ; Math # L& [2] GREEK CAPITAL THETA SYMBOL..GREEK LUNATE EPSILON SYMBOL +03F6 ; Math # Sm GREEK REVERSED LUNATE EPSILON SYMBOL +0606..0608 ; Math # Sm [3] ARABIC-INDIC CUBE ROOT..ARABIC RAY +2016 ; Math # Po DOUBLE VERTICAL LINE +2032..2034 ; Math # Po [3] PRIME..TRIPLE PRIME +2040 ; Math # Pc CHARACTER TIE +2044 ; Math # Sm FRACTION SLASH +2052 ; Math # Sm COMMERCIAL MINUS SIGN +2061..2064 ; Math # Cf [4] FUNCTION APPLICATION..INVISIBLE PLUS +207A..207C ; Math # Sm [3] SUPERSCRIPT PLUS SIGN..SUPERSCRIPT EQUALS SIGN +207D ; Math # Ps SUPERSCRIPT LEFT PARENTHESIS +207E ; Math # Pe SUPERSCRIPT RIGHT PARENTHESIS +208A..208C ; Math # Sm [3] SUBSCRIPT PLUS SIGN..SUBSCRIPT EQUALS SIGN +208D ; Math # Ps SUBSCRIPT LEFT PARENTHESIS +208E ; Math # Pe SUBSCRIPT RIGHT PARENTHESIS +20D0..20DC ; Math # Mn [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE +20E1 ; Math # Mn COMBINING LEFT RIGHT ARROW ABOVE +20E5..20E6 ; Math # Mn [2] COMBINING REVERSE SOLIDUS OVERLAY..COMBINING DOUBLE VERTICAL STROKE OVERLAY +20EB..20EF ; Math # Mn [5] COMBINING LONG DOUBLE SOLIDUS OVERLAY..COMBINING RIGHT ARROW BELOW +2102 ; Math # L& DOUBLE-STRUCK CAPITAL C +2107 ; Math # L& EULER CONSTANT +210A..2113 ; Math # L& [10] SCRIPT SMALL G..SCRIPT SMALL L +2115 ; Math # L& DOUBLE-STRUCK CAPITAL N +2118 ; Math # Sm SCRIPT CAPITAL P +2119..211D ; Math # L& [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R +2124 ; Math # L& DOUBLE-STRUCK CAPITAL Z +2128 ; Math # L& BLACK-LETTER CAPITAL Z +2129 ; Math # So TURNED GREEK SMALL LETTER IOTA +212C..212D ; Math # L& [2] SCRIPT CAPITAL B..BLACK-LETTER CAPITAL C +212F..2131 ; Math # L& [3] SCRIPT SMALL E..SCRIPT CAPITAL F +2133..2134 ; Math # L& [2] SCRIPT CAPITAL M..SCRIPT SMALL O +2135..2138 ; Math # Lo [4] ALEF SYMBOL..DALET SYMBOL +213C..213F ; Math # L& [4] DOUBLE-STRUCK SMALL PI..DOUBLE-STRUCK CAPITAL PI +2140..2144 ; Math # Sm [5] DOUBLE-STRUCK N-ARY SUMMATION..TURNED SANS-SERIF CAPITAL Y +2145..2149 ; Math # L& [5] DOUBLE-STRUCK ITALIC CAPITAL D..DOUBLE-STRUCK ITALIC SMALL J +214B ; Math # Sm TURNED AMPERSAND +2190..2194 ; Math # Sm [5] LEFTWARDS ARROW..LEFT RIGHT ARROW +2195..2199 ; Math # So [5] UP DOWN ARROW..SOUTH WEST ARROW +219A..219B ; Math # Sm [2] LEFTWARDS ARROW WITH STROKE..RIGHTWARDS ARROW WITH STROKE +219C..219F ; Math # So [4] LEFTWARDS WAVE ARROW..UPWARDS TWO HEADED ARROW +21A0 ; Math # Sm RIGHTWARDS TWO HEADED ARROW +21A1..21A2 ; Math # So [2] DOWNWARDS TWO HEADED ARROW..LEFTWARDS ARROW WITH TAIL +21A3 ; Math # Sm RIGHTWARDS ARROW WITH TAIL +21A4..21A5 ; Math # So [2] LEFTWARDS ARROW FROM BAR..UPWARDS ARROW FROM BAR +21A6 ; Math # Sm RIGHTWARDS ARROW FROM BAR +21A7 ; Math # So DOWNWARDS ARROW FROM BAR +21A9..21AD ; Math # So [5] LEFTWARDS ARROW WITH HOOK..LEFT RIGHT WAVE ARROW +21AE ; Math # Sm LEFT RIGHT ARROW WITH STROKE +21B0..21B1 ; Math # So [2] UPWARDS ARROW WITH TIP LEFTWARDS..UPWARDS ARROW WITH TIP RIGHTWARDS +21B6..21B7 ; Math # So [2] ANTICLOCKWISE TOP SEMICIRCLE ARROW..CLOCKWISE TOP SEMICIRCLE ARROW +21BC..21CD ; Math # So [18] LEFTWARDS HARPOON WITH BARB UPWARDS..LEFTWARDS DOUBLE ARROW WITH STROKE +21CE..21CF ; Math # Sm [2] LEFT RIGHT DOUBLE ARROW WITH STROKE..RIGHTWARDS DOUBLE ARROW WITH STROKE +21D0..21D1 ; Math # So [2] LEFTWARDS DOUBLE ARROW..UPWARDS DOUBLE ARROW +21D2 ; Math # Sm RIGHTWARDS DOUBLE ARROW +21D3 ; Math # So DOWNWARDS DOUBLE ARROW +21D4 ; Math # Sm LEFT RIGHT DOUBLE ARROW +21D5..21DB ; Math # So [7] UP DOWN DOUBLE ARROW..RIGHTWARDS TRIPLE ARROW +21DD ; Math # So RIGHTWARDS SQUIGGLE ARROW +21E4..21E5 ; Math # So [2] LEFTWARDS ARROW TO BAR..RIGHTWARDS ARROW TO BAR +21F4..22FF ; Math # Sm [268] RIGHT ARROW WITH SMALL CIRCLE..Z NOTATION BAG MEMBERSHIP +2308 ; Math # Ps LEFT CEILING +2309 ; Math # Pe RIGHT CEILING +230A ; Math # Ps LEFT FLOOR +230B ; Math # Pe RIGHT FLOOR +2320..2321 ; Math # Sm [2] TOP HALF INTEGRAL..BOTTOM HALF INTEGRAL +237C ; Math # Sm RIGHT ANGLE WITH DOWNWARDS ZIGZAG ARROW +239B..23B3 ; Math # Sm [25] LEFT PARENTHESIS UPPER HOOK..SUMMATION BOTTOM +23B4..23B5 ; Math # So [2] TOP SQUARE BRACKET..BOTTOM SQUARE BRACKET +23B7 ; Math # So RADICAL SYMBOL BOTTOM +23D0 ; Math # So VERTICAL LINE EXTENSION +23DC..23E1 ; Math # Sm [6] TOP PARENTHESIS..BOTTOM TORTOISE SHELL BRACKET +23E2 ; Math # So WHITE TRAPEZIUM +25A0..25A1 ; Math # So [2] BLACK SQUARE..WHITE SQUARE +25AE..25B6 ; Math # So [9] BLACK VERTICAL RECTANGLE..BLACK RIGHT-POINTING TRIANGLE +25B7 ; Math # Sm WHITE RIGHT-POINTING TRIANGLE +25BC..25C0 ; Math # So [5] BLACK DOWN-POINTING TRIANGLE..BLACK LEFT-POINTING TRIANGLE +25C1 ; Math # Sm WHITE LEFT-POINTING TRIANGLE +25C6..25C7 ; Math # So [2] BLACK DIAMOND..WHITE DIAMOND +25CA..25CB ; Math # So [2] LOZENGE..WHITE CIRCLE +25CF..25D3 ; Math # So [5] BLACK CIRCLE..CIRCLE WITH UPPER HALF BLACK +25E2 ; Math # So BLACK LOWER RIGHT TRIANGLE +25E4 ; Math # So BLACK UPPER LEFT TRIANGLE +25E7..25EC ; Math # So [6] SQUARE WITH LEFT HALF BLACK..WHITE UP-POINTING TRIANGLE WITH DOT +25F8..25FF ; Math # Sm [8] UPPER LEFT TRIANGLE..LOWER RIGHT TRIANGLE +2605..2606 ; Math # So [2] BLACK STAR..WHITE STAR +2640 ; Math # So FEMALE SIGN +2642 ; Math # So MALE SIGN +2660..2663 ; Math # So [4] BLACK SPADE SUIT..BLACK CLUB SUIT +266D..266E ; Math # So [2] MUSIC FLAT SIGN..MUSIC NATURAL SIGN +266F ; Math # Sm MUSIC SHARP SIGN +27C0..27C4 ; Math # Sm [5] THREE DIMENSIONAL ANGLE..OPEN SUPERSET +27C5 ; Math # Ps LEFT S-SHAPED BAG DELIMITER +27C6 ; Math # Pe RIGHT S-SHAPED BAG DELIMITER +27C7..27E5 ; Math # Sm [31] OR WITH DOT INSIDE..WHITE SQUARE WITH RIGHTWARDS TICK +27E6 ; Math # Ps MATHEMATICAL LEFT WHITE SQUARE BRACKET +27E7 ; Math # Pe MATHEMATICAL RIGHT WHITE SQUARE BRACKET +27E8 ; Math # Ps MATHEMATICAL LEFT ANGLE BRACKET +27E9 ; Math # Pe MATHEMATICAL RIGHT ANGLE BRACKET +27EA ; Math # Ps MATHEMATICAL LEFT DOUBLE ANGLE BRACKET +27EB ; Math # Pe MATHEMATICAL RIGHT DOUBLE ANGLE BRACKET +27EC ; Math # Ps MATHEMATICAL LEFT WHITE TORTOISE SHELL BRACKET +27ED ; Math # Pe MATHEMATICAL RIGHT WHITE TORTOISE SHELL BRACKET +27EE ; Math # Ps MATHEMATICAL LEFT FLATTENED PARENTHESIS +27EF ; Math # Pe MATHEMATICAL RIGHT FLATTENED PARENTHESIS +27F0..27FF ; Math # Sm [16] UPWARDS QUADRUPLE ARROW..LONG RIGHTWARDS SQUIGGLE ARROW +2900..2982 ; Math # Sm [131] RIGHTWARDS TWO-HEADED ARROW WITH VERTICAL STROKE..Z NOTATION TYPE COLON +2983 ; Math # Ps LEFT WHITE CURLY BRACKET +2984 ; Math # Pe RIGHT WHITE CURLY BRACKET +2985 ; Math # Ps LEFT WHITE PARENTHESIS +2986 ; Math # Pe RIGHT WHITE PARENTHESIS +2987 ; Math # Ps Z NOTATION LEFT IMAGE BRACKET +2988 ; Math # Pe Z NOTATION RIGHT IMAGE BRACKET +2989 ; Math # Ps Z NOTATION LEFT BINDING BRACKET +298A ; Math # Pe Z NOTATION RIGHT BINDING BRACKET +298B ; Math # Ps LEFT SQUARE BRACKET WITH UNDERBAR +298C ; Math # Pe RIGHT SQUARE BRACKET WITH UNDERBAR +298D ; Math # Ps LEFT SQUARE BRACKET WITH TICK IN TOP CORNER +298E ; Math # Pe RIGHT SQUARE BRACKET WITH TICK IN BOTTOM CORNER +298F ; Math # Ps LEFT SQUARE BRACKET WITH TICK IN BOTTOM CORNER +2990 ; Math # Pe RIGHT SQUARE BRACKET WITH TICK IN TOP CORNER +2991 ; Math # Ps LEFT ANGLE BRACKET WITH DOT +2992 ; Math # Pe RIGHT ANGLE BRACKET WITH DOT +2993 ; Math # Ps LEFT ARC LESS-THAN BRACKET +2994 ; Math # Pe RIGHT ARC GREATER-THAN BRACKET +2995 ; Math # Ps DOUBLE LEFT ARC GREATER-THAN BRACKET +2996 ; Math # Pe DOUBLE RIGHT ARC LESS-THAN BRACKET +2997 ; Math # Ps LEFT BLACK TORTOISE SHELL BRACKET +2998 ; Math # Pe RIGHT BLACK TORTOISE SHELL BRACKET +2999..29D7 ; Math # Sm [63] DOTTED FENCE..BLACK HOURGLASS +29D8 ; Math # Ps LEFT WIGGLY FENCE +29D9 ; Math # Pe RIGHT WIGGLY FENCE +29DA ; Math # Ps LEFT DOUBLE WIGGLY FENCE +29DB ; Math # Pe RIGHT DOUBLE WIGGLY FENCE +29DC..29FB ; Math # Sm [32] INCOMPLETE INFINITY..TRIPLE PLUS +29FC ; Math # Ps LEFT-POINTING CURVED ANGLE BRACKET +29FD ; Math # Pe RIGHT-POINTING CURVED ANGLE BRACKET +29FE..2AFF ; Math # Sm [258] TINY..N-ARY WHITE VERTICAL BAR +2B30..2B44 ; Math # Sm [21] LEFT ARROW WITH SMALL CIRCLE..RIGHTWARDS ARROW THROUGH SUPERSET +2B47..2B4C ; Math # Sm [6] REVERSE TILDE OPERATOR ABOVE RIGHTWARDS ARROW..RIGHTWARDS ARROW ABOVE REVERSE TILDE OPERATOR +FB29 ; Math # Sm HEBREW LETTER ALTERNATIVE PLUS SIGN +FE61 ; Math # Po SMALL ASTERISK +FE62 ; Math # Sm SMALL PLUS SIGN +FE63 ; Math # Pd SMALL HYPHEN-MINUS +FE64..FE66 ; Math # Sm [3] SMALL LESS-THAN SIGN..SMALL EQUALS SIGN +FE68 ; Math # Po SMALL REVERSE SOLIDUS +FF0B ; Math # Sm FULLWIDTH PLUS SIGN +FF1C..FF1E ; Math # Sm [3] FULLWIDTH LESS-THAN SIGN..FULLWIDTH GREATER-THAN SIGN +FF3C ; Math # Po FULLWIDTH REVERSE SOLIDUS +FF3E ; Math # Sk FULLWIDTH CIRCUMFLEX ACCENT +FF5C ; Math # Sm FULLWIDTH VERTICAL LINE +FF5E ; Math # Sm FULLWIDTH TILDE +FFE2 ; Math # Sm FULLWIDTH NOT SIGN +FFE9..FFEC ; Math # Sm [4] HALFWIDTH LEFTWARDS ARROW..HALFWIDTH DOWNWARDS ARROW +1D400..1D454 ; Math # L& [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G +1D456..1D49C ; Math # L& [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A +1D49E..1D49F ; Math # L& [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D +1D4A2 ; Math # L& MATHEMATICAL SCRIPT CAPITAL G +1D4A5..1D4A6 ; Math # L& [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K +1D4A9..1D4AC ; Math # L& [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q +1D4AE..1D4B9 ; Math # L& [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D +1D4BB ; Math # L& MATHEMATICAL SCRIPT SMALL F +1D4BD..1D4C3 ; Math # L& [7] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL N +1D4C5..1D505 ; Math # L& [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B +1D507..1D50A ; Math # L& [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G +1D50D..1D514 ; Math # L& [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q +1D516..1D51C ; Math # L& [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y +1D51E..1D539 ; Math # L& [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B +1D53B..1D53E ; Math # L& [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G +1D540..1D544 ; Math # L& [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M +1D546 ; Math # L& MATHEMATICAL DOUBLE-STRUCK CAPITAL O +1D54A..1D550 ; Math # L& [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y +1D552..1D6A5 ; Math # L& [340] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL ITALIC SMALL DOTLESS J +1D6A8..1D6C0 ; Math # L& [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA +1D6C1 ; Math # Sm MATHEMATICAL BOLD NABLA +1D6C2..1D6DA ; Math # L& [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA +1D6DB ; Math # Sm MATHEMATICAL BOLD PARTIAL DIFFERENTIAL +1D6DC..1D6FA ; Math # L& [31] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL ITALIC CAPITAL OMEGA +1D6FB ; Math # Sm MATHEMATICAL ITALIC NABLA +1D6FC..1D714 ; Math # L& [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA +1D715 ; Math # Sm MATHEMATICAL ITALIC PARTIAL DIFFERENTIAL +1D716..1D734 ; Math # L& [31] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA +1D735 ; Math # Sm MATHEMATICAL BOLD ITALIC NABLA +1D736..1D74E ; Math # L& [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA +1D74F ; Math # Sm MATHEMATICAL BOLD ITALIC PARTIAL DIFFERENTIAL +1D750..1D76E ; Math # L& [31] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA +1D76F ; Math # Sm MATHEMATICAL SANS-SERIF BOLD NABLA +1D770..1D788 ; Math # L& [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA +1D789 ; Math # Sm MATHEMATICAL SANS-SERIF BOLD PARTIAL DIFFERENTIAL +1D78A..1D7A8 ; Math # L& [31] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA +1D7A9 ; Math # Sm MATHEMATICAL SANS-SERIF BOLD ITALIC NABLA +1D7AA..1D7C2 ; Math # L& [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA +1D7C3 ; Math # Sm MATHEMATICAL SANS-SERIF BOLD ITALIC PARTIAL DIFFERENTIAL +1D7C4..1D7CB ; Math # L& [8] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD SMALL DIGAMMA +1D7CE..1D7FF ; Math # Nd [50] MATHEMATICAL BOLD DIGIT ZERO..MATHEMATICAL MONOSPACE DIGIT NINE +1EE00..1EE03 ; Math # Lo [4] ARABIC MATHEMATICAL ALEF..ARABIC MATHEMATICAL DAL +1EE05..1EE1F ; Math # Lo [27] ARABIC MATHEMATICAL WAW..ARABIC MATHEMATICAL DOTLESS QAF +1EE21..1EE22 ; Math # Lo [2] ARABIC MATHEMATICAL INITIAL BEH..ARABIC MATHEMATICAL INITIAL JEEM +1EE24 ; Math # Lo ARABIC MATHEMATICAL INITIAL HEH +1EE27 ; Math # Lo ARABIC MATHEMATICAL INITIAL HAH +1EE29..1EE32 ; Math # Lo [10] ARABIC MATHEMATICAL INITIAL YEH..ARABIC MATHEMATICAL INITIAL QAF +1EE34..1EE37 ; Math # Lo [4] ARABIC MATHEMATICAL INITIAL SHEEN..ARABIC MATHEMATICAL INITIAL KHAH +1EE39 ; Math # Lo ARABIC MATHEMATICAL INITIAL DAD +1EE3B ; Math # Lo ARABIC MATHEMATICAL INITIAL GHAIN +1EE42 ; Math # Lo ARABIC MATHEMATICAL TAILED JEEM +1EE47 ; Math # Lo ARABIC MATHEMATICAL TAILED HAH +1EE49 ; Math # Lo ARABIC MATHEMATICAL TAILED YEH +1EE4B ; Math # Lo ARABIC MATHEMATICAL TAILED LAM +1EE4D..1EE4F ; Math # Lo [3] ARABIC MATHEMATICAL TAILED NOON..ARABIC MATHEMATICAL TAILED AIN +1EE51..1EE52 ; Math # Lo [2] ARABIC MATHEMATICAL TAILED SAD..ARABIC MATHEMATICAL TAILED QAF +1EE54 ; Math # Lo ARABIC MATHEMATICAL TAILED SHEEN +1EE57 ; Math # Lo ARABIC MATHEMATICAL TAILED KHAH +1EE59 ; Math # Lo ARABIC MATHEMATICAL TAILED DAD +1EE5B ; Math # Lo ARABIC MATHEMATICAL TAILED GHAIN +1EE5D ; Math # Lo ARABIC MATHEMATICAL TAILED DOTLESS NOON +1EE5F ; Math # Lo ARABIC MATHEMATICAL TAILED DOTLESS QAF +1EE61..1EE62 ; Math # Lo [2] ARABIC MATHEMATICAL STRETCHED BEH..ARABIC MATHEMATICAL STRETCHED JEEM +1EE64 ; Math # Lo ARABIC MATHEMATICAL STRETCHED HEH +1EE67..1EE6A ; Math # Lo [4] ARABIC MATHEMATICAL STRETCHED HAH..ARABIC MATHEMATICAL STRETCHED KAF +1EE6C..1EE72 ; Math # Lo [7] ARABIC MATHEMATICAL STRETCHED MEEM..ARABIC MATHEMATICAL STRETCHED QAF +1EE74..1EE77 ; Math # Lo [4] ARABIC MATHEMATICAL STRETCHED SHEEN..ARABIC MATHEMATICAL STRETCHED KHAH +1EE79..1EE7C ; Math # Lo [4] ARABIC MATHEMATICAL STRETCHED DAD..ARABIC MATHEMATICAL STRETCHED DOTLESS BEH +1EE7E ; Math # Lo ARABIC MATHEMATICAL STRETCHED DOTLESS FEH +1EE80..1EE89 ; Math # Lo [10] ARABIC MATHEMATICAL LOOPED ALEF..ARABIC MATHEMATICAL LOOPED YEH +1EE8B..1EE9B ; Math # Lo [17] ARABIC MATHEMATICAL LOOPED LAM..ARABIC MATHEMATICAL LOOPED GHAIN +1EEA1..1EEA3 ; Math # Lo [3] ARABIC MATHEMATICAL DOUBLE-STRUCK BEH..ARABIC MATHEMATICAL DOUBLE-STRUCK DAL +1EEA5..1EEA9 ; Math # Lo [5] ARABIC MATHEMATICAL DOUBLE-STRUCK WAW..ARABIC MATHEMATICAL DOUBLE-STRUCK YEH +1EEAB..1EEBB ; Math # Lo [17] ARABIC MATHEMATICAL DOUBLE-STRUCK LAM..ARABIC MATHEMATICAL DOUBLE-STRUCK GHAIN +1EEF0..1EEF1 ; Math # Sm [2] ARABIC MATHEMATICAL OPERATOR MEEM WITH HAH WITH TATWEEL..ARABIC MATHEMATICAL OPERATOR HAH WITH DAL + +# Total code points: 2310 + +# ================================================ + +# Derived Property: Alphabetic +# Generated from: Uppercase + Lowercase + Lt + Lm + Lo + Nl + Other_Alphabetic + +0041..005A ; Alphabetic # L& [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z +0061..007A ; Alphabetic # L& [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z +00AA ; Alphabetic # Lo FEMININE ORDINAL INDICATOR +00B5 ; Alphabetic # L& MICRO SIGN +00BA ; Alphabetic # Lo MASCULINE ORDINAL INDICATOR +00C0..00D6 ; Alphabetic # L& [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS +00D8..00F6 ; Alphabetic # L& [31] LATIN CAPITAL LETTER O WITH STROKE..LATIN SMALL LETTER O WITH DIAERESIS +00F8..01BA ; Alphabetic # L& [195] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER EZH WITH TAIL +01BB ; Alphabetic # Lo LATIN LETTER TWO WITH STROKE +01BC..01BF ; Alphabetic # L& [4] LATIN CAPITAL LETTER TONE FIVE..LATIN LETTER WYNN +01C0..01C3 ; Alphabetic # Lo [4] LATIN LETTER DENTAL CLICK..LATIN LETTER RETROFLEX CLICK +01C4..0293 ; Alphabetic # L& [208] LATIN CAPITAL LETTER DZ WITH CARON..LATIN SMALL LETTER EZH WITH CURL +0294 ; Alphabetic # Lo LATIN LETTER GLOTTAL STOP +0295..02AF ; Alphabetic # L& [27] LATIN LETTER PHARYNGEAL VOICED FRICATIVE..LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL +02B0..02C1 ; Alphabetic # Lm [18] MODIFIER LETTER SMALL H..MODIFIER LETTER REVERSED GLOTTAL STOP +02C6..02D1 ; Alphabetic # Lm [12] MODIFIER LETTER CIRCUMFLEX ACCENT..MODIFIER LETTER HALF TRIANGULAR COLON +02E0..02E4 ; Alphabetic # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +02EC ; Alphabetic # Lm MODIFIER LETTER VOICING +02EE ; Alphabetic # Lm MODIFIER LETTER DOUBLE APOSTROPHE +0345 ; Alphabetic # Mn COMBINING GREEK YPOGEGRAMMENI From pypy.commits at gmail.com Thu Jan 4 18:25:26 2018 From: pypy.commits at gmail.com (amauryfa) Date: Thu, 04 Jan 2018 15:25:26 -0800 (PST) Subject: [pypy-commit] pypy py3.6: Upgrade pypy3.6 to UnicodeData 9.0.0 Message-ID: <5a4eb7e6.12badf0a.57715.5348@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r93627:c06ae9f3ddac Date: 2018-01-04 17:58 +0100 http://bitbucket.org/pypy/pypy/changeset/c06ae9f3ddac/ Log: Upgrade pypy3.6 to UnicodeData 9.0.0 diff --git a/pypy/module/unicodedata/__init__.py b/pypy/module/unicodedata/__init__.py --- a/pypy/module/unicodedata/__init__.py +++ b/pypy/module/unicodedata/__init__.py @@ -15,7 +15,7 @@ interpleveldefs = { 'unidata_version' : 'space.wrap(interp_ucd.ucd.version)', 'ucd_3_2_0' : 'space.wrap(interp_ucd.ucd_3_2_0)', - 'ucd_8_0_0' : 'space.wrap(interp_ucd.ucd_8_0_0)', + 'ucd_9_0_0' : 'space.wrap(interp_ucd.ucd_9_0_0)', 'ucd' : 'space.wrap(interp_ucd.ucd)', '__doc__' : "space.wrap('unicode character database')", } diff --git a/pypy/module/unicodedata/interp_ucd.py b/pypy/module/unicodedata/interp_ucd.py --- a/pypy/module/unicodedata/interp_ucd.py +++ b/pypy/module/unicodedata/interp_ucd.py @@ -9,7 +9,7 @@ from rpython.rlib.rarithmetic import r_longlong from rpython.rlib.objectmodel import we_are_translated from rpython.rlib.runicode import MAXUNICODE -from rpython.rlib.unicodedata import unicodedb_8_0_0, unicodedb_3_2_0 +from rpython.rlib.unicodedata import unicodedb_9_0_0, unicodedb_3_2_0 from rpython.rlib.runicode import code_to_unichr, ord_accepts_surrogate import sys @@ -337,5 +337,5 @@ **methods) ucd_3_2_0 = UCD(unicodedb_3_2_0) -ucd_8_0_0 = UCD(unicodedb_8_0_0) -ucd = ucd_8_0_0 +ucd_9_0_0 = UCD(unicodedb_9_0_0) +ucd = ucd_9_0_0 diff --git a/pypy/module/unicodedata/test/test_unicodedata.py b/pypy/module/unicodedata/test/test_unicodedata.py --- a/pypy/module/unicodedata/test/test_unicodedata.py +++ b/pypy/module/unicodedata/test/test_unicodedata.py @@ -153,3 +153,9 @@ for cp in range(0xf0000, 0xf0300, 7): exc = raises(ValueError, unicodedata.name, chr(cp)) assert str(exc.value) == 'no such name' + + def test_east_asian_width_9_0_changes(self): + import unicodedata + assert unicodedata.ucd_3_2_0.east_asian_width('\u231a') == 'N' + assert unicodedata.ucd.east_asian_width('\u231a') == 'W' + From pypy.commits at gmail.com Fri Jan 5 15:17:55 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 05 Jan 2018 12:17:55 -0800 (PST) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <5a4fdd73.06c4df0a.d303d.18b2@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r93628:799e8d00a8ec Date: 2018-01-05 20:17 +0000 http://bitbucket.org/pypy/pypy/changeset/799e8d00a8ec/ Log: hg merge default diff too long, truncating to 2000 out of 4420 lines diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -46,3 +46,7 @@ 84a2f3e6a7f88f2fe698e473998755b3bd1a12e2 release-pypy2.7-v5.9.0 0e7ea4fe15e82d5124e805e2e4a37cae1a402d4b release-pypy2.7-v5.10.0 a91df6163fb76df245091f741dbf6a23ddc72374 release-pypy3.5-v5.10.0 +a91df6163fb76df245091f741dbf6a23ddc72374 release-pypy3.5-v5.10.0 +0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0 +0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0 +09f9160b643e3f02ccb8c843b2fbb4e5cbf54082 release-pypy3.5-v5.10.0 diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -339,8 +339,10 @@ Stanisław Halik Julien Phalip Roman Podoliaka + Steve Papanik Eli Stevens Boglarka Vezer + gabrielg PavloKapyshin Tomer Chachamu Christopher Groskopf @@ -363,11 +365,13 @@ Konrad Delong Dinu Gherman pizi + Tomáš Pružina James Robert Armin Ronacher Diana Popa Mads Kiilerich Brett Cannon + Caleb Hattingh aliceinwire Zooko Wilcox-O Hearn James Lan @@ -388,6 +392,7 @@ Jason Madden Yaroslav Fedevych Even Wiik Thomassen + m at funkyhat.org Stefan Marr Heinrich-Heine University, Germany diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py --- a/lib_pypy/_ctypes/array.py +++ b/lib_pypy/_ctypes/array.py @@ -12,7 +12,8 @@ if cls == (_CData,): # this is the Array class defined below res._ffiarray = None return res - if not hasattr(res, '_length_') or not isinstance(res._length_, int): + if not hasattr(res, '_length_') or not isinstance(res._length_, + (int, long)): raise AttributeError( "class must define a '_length_' attribute, " "which must be a positive integer") diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -217,6 +217,7 @@ Alejandro J. Cura Vladimir Kryachko Gabriel + Thomas Hisch Mark Williams Kunal Grover Nathan Taylor @@ -306,8 +307,10 @@ Stanisław Halik Julien Phalip Roman Podoliaka + Steve Papanik Eli Stevens Boglarka Vezer + gabrielg PavloKapyshin Tomer Chachamu Christopher Groskopf @@ -330,11 +333,13 @@ Konrad Delong Dinu Gherman pizi + Tomáš Pružina James Robert Armin Ronacher Diana Popa Mads Kiilerich Brett Cannon + Caleb Hattingh aliceinwire Zooko Wilcox-O Hearn James Lan @@ -355,4 +360,5 @@ Jason Madden Yaroslav Fedevych Even Wiik Thomassen + m at funkyhat.org Stefan Marr diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -558,6 +558,15 @@ ``del foo.bar`` where ``foo`` is a module (or class) that contains the function ``bar``, is significantly slower than CPython. +* Various built-in functions in CPython accept only positional arguments + and not keyword arguments. That can be considered a long-running + historical detail: newer functions tend to accept keyword arguments + and older function are occasionally fixed to do so as well. In PyPy, + most built-in functions accept keyword arguments (``help()`` shows the + argument names). But don't rely on it too much because future + versions of PyPy may have to rename the arguments if CPython starts + accepting them too. + .. _`is ignored in PyPy`: http://bugs.python.org/issue14621 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/ diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py --- a/pypy/doc/tool/makecontributor.py +++ b/pypy/doc/tool/makecontributor.py @@ -81,6 +81,7 @@ 'Yasir Suhail':['yasirs'], 'Squeaky': ['squeaky'], "Amaury Forgeot d'Arc": ['amauryfa at gmail.com'], + "Dodan Mihai": ['mihai.dodan at gmail.com'], } alias_map = {} diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,3 +5,8 @@ .. this is a revision shortly after release-pypy2.7-v5.10.0 .. startrev: 6b024edd9d12 +.. branch: cpyext-avoid-roundtrip + +Big refactoring of some cpyext code, which avoids a lot of nonsense when +calling C from Python and vice-versa: the result is a big speedup in +function/method calls, up to 6 times faster. diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py --- a/pypy/module/_cppyy/converter.py +++ b/pypy/module/_cppyy/converter.py @@ -682,8 +682,8 @@ if hasattr(space, "fake"): raise NotImplementedError space.getbuiltinmodule("cpyext") - from pypy.module.cpyext.pyobject import Py_DecRef, PyObject - Py_DecRef(space, rffi.cast(PyObject, rffi.cast(rffi.VOIDPP, arg)[0])) + from pypy.module.cpyext.pyobject import decref, PyObject + decref(space, rffi.cast(PyObject, rffi.cast(rffi.VOIDPP, arg)[0])) class MacroConverter(TypeConverter): diff --git a/pypy/module/_cppyy/executor.py b/pypy/module/_cppyy/executor.py --- a/pypy/module/_cppyy/executor.py +++ b/pypy/module/_cppyy/executor.py @@ -224,11 +224,11 @@ def wrap_result(self, space, lresult): space.getbuiltinmodule("cpyext") - from pypy.module.cpyext.pyobject import PyObject, from_ref, make_ref, Py_DecRef + from pypy.module.cpyext.pyobject import PyObject, from_ref, make_ref, decref result = rffi.cast(PyObject, lresult) w_obj = from_ref(space, result) if result: - Py_DecRef(space, result) + decref(space, result) return w_obj def execute(self, space, cppmethod, cppthis, num_args, args): diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py --- a/pypy/module/cpyext/__init__.py +++ b/pypy/module/cpyext/__init__.py @@ -16,11 +16,12 @@ space.fromcache(State).startup(space) method = pypy.module.cpyext.typeobject.get_new_method_def(space) # the w_self argument here is a dummy, the only thing done with w_obj - # is call space.type on it - w_obj = pypy.module.cpyext.methodobject.W_PyCFunctionObject(space, method, space.w_None) - space.appexec([space.type(w_obj)], """(methodtype): + # is call type() on it + w_obj = pypy.module.cpyext.methodobject.W_PyCFunctionObject(space, + method, space.w_None) + space.appexec([w_obj], """(meth): from pickle import Pickler - Pickler.dispatch[methodtype] = Pickler.save_global + Pickler.dispatch[type(meth)] = Pickler.save_global """) def register_atexit(self, function): diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -206,6 +206,9 @@ # id. Invariant: this variable always contain 0 when the PyPy GIL is # released. It should also contain 0 when regular RPython code # executes. In non-cpyext-related code, it will thus always be 0. +# When cpyext-related C code runs, it contains the thread id (usually) +# or the value -1 (only for state.C.PyXxx() functions which are short- +# running and should not themselves release the GIL). # # **make_generic_cpy_call():** RPython to C, with the GIL held. Before # the call, must assert that the global variable is 0 and set the @@ -257,14 +260,73 @@ cpyext_namespace = NameManager('cpyext_') -class ApiFunction(object): - def __init__(self, argtypes, restype, callable, error=CANNOT_FAIL, - c_name=None, cdecl=None, gil=None, - result_borrowed=False, result_is_ll=False): +class BaseApiFunction(object): + def __init__(self, argtypes, restype, callable): self.argtypes = argtypes self.restype = restype self.functype = lltype.Ptr(lltype.FuncType(argtypes, restype)) self.callable = callable + self.cdecl = None # default + # + def get_llhelper(space): + return llhelper(self.functype, self.get_wrapper(space)) + self.get_llhelper = get_llhelper + + def get_api_decl(self, name, c_writer): + restype = self.get_c_restype(c_writer) + args = self.get_c_args(c_writer) + res = self.API_VISIBILITY % (restype,) + return "{res} {name}({args});".format(**locals()) + + def get_c_restype(self, c_writer): + if self.cdecl: + return self.cdecl.tp.result.get_c_name() + return c_writer.gettype(self.restype).replace('@', '').strip() + + def get_c_args(self, c_writer): + if self.cdecl: + args = [tp.get_c_name('arg%d' % i) for i, tp in + enumerate(self.cdecl.tp.args)] + return ', '.join(args) or "void" + args = [] + for i, argtype in enumerate(self.argtypes): + if argtype is CONST_STRING: + arg = 'const char *@' + elif argtype is CONST_STRINGP: + arg = 'const char **@' + elif argtype is CONST_WSTRING: + arg = 'const wchar_t *@' + else: + arg = c_writer.gettype(argtype) + arg = arg.replace('@', 'arg%d' % (i,)).strip() + args.append(arg) + args = ', '.join(args) or "void" + return args + + def get_ptr_decl(self, name, c_writer): + restype = self.get_c_restype(c_writer) + args = self.get_c_args(c_writer) + return "{restype} (*{name})({args});".format(**locals()) + + def get_ctypes_impl(self, name, c_writer): + restype = self.get_c_restype(c_writer) + args = self.get_c_args(c_writer) + callargs = ', '.join('arg%d' % (i,) + for i in range(len(self.argtypes))) + if self.restype is lltype.Void: + body = "{ _pypyAPI.%s(%s); }" % (name, callargs) + else: + body = "{ return _pypyAPI.%s(%s); }" % (name, callargs) + return '%s %s(%s)\n%s' % (restype, name, args, body) + + +class ApiFunction(BaseApiFunction): + API_VISIBILITY = "PyAPI_FUNC(%s)" + + def __init__(self, argtypes, restype, callable, error=CANNOT_FAIL, + c_name=None, cdecl=None, gil=None, + result_borrowed=False, result_is_ll=False): + BaseApiFunction.__init__(self, argtypes, restype, callable) self.error_value = error self.c_name = c_name self.cdecl = cdecl @@ -282,10 +344,6 @@ self.gil = gil self.result_borrowed = result_borrowed self.result_is_ll = result_is_ll - # - def get_llhelper(space): - return llhelper(self.functype, self.get_wrapper(space)) - self.get_llhelper = get_llhelper def __repr__(self): return "" % (self.callable.__name__,) @@ -383,57 +441,17 @@ arg = input_arg newargs += (arg, ) try: - return self.callable(space, *newargs) + result = self.callable(space, *newargs) finally: keepalive_until_here(*keepalives) + # + # this is just a sanity check to ensure that we don't forget to + # specify result_is_ll=True + if self.restype == PyObject: + assert self.result_is_ll == is_pyobj(result) + return result return unwrapper - def get_c_restype(self, c_writer): - if self.cdecl: - return self.cdecl.tp.result.get_c_name() - return c_writer.gettype(self.restype).replace('@', '').strip() - - def get_c_args(self, c_writer): - if self.cdecl: - args = [tp.get_c_name('arg%d' % i) for i, tp in - enumerate(self.cdecl.tp.args)] - return ', '.join(args) or "void" - args = [] - for i, argtype in enumerate(self.argtypes): - if argtype is CONST_STRING: - arg = 'const char *@' - elif argtype is CONST_STRINGP: - arg = 'const char **@' - elif argtype is CONST_WSTRING: - arg = 'const wchar_t *@' - else: - arg = c_writer.gettype(argtype) - arg = arg.replace('@', 'arg%d' % (i,)).strip() - args.append(arg) - args = ', '.join(args) or "void" - return args - - def get_api_decl(self, name, c_writer): - restype = self.get_c_restype(c_writer) - args = self.get_c_args(c_writer) - return "PyAPI_FUNC({restype}) {name}({args});".format(**locals()) - - def get_ptr_decl(self, name, c_writer): - restype = self.get_c_restype(c_writer) - args = self.get_c_args(c_writer) - return "{restype} (*{name})({args});".format(**locals()) - - def get_ctypes_impl(self, name, c_writer): - restype = self.get_c_restype(c_writer) - args = self.get_c_args(c_writer) - callargs = ', '.join('arg%d' % (i,) - for i in range(len(self.argtypes))) - if self.restype is lltype.Void: - body = "{ _pypyAPI.%s(%s); }" % (name, callargs) - else: - body = "{ return _pypyAPI.%s(%s); }" % (name, callargs) - return '%s %s(%s)\n%s' % (restype, name, args, body) - DEFAULT_HEADER = 'pypy_decl.h' def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, header=DEFAULT_HEADER, @@ -471,6 +489,26 @@ return unwrapper return decorate +class COnlyApiFunction(BaseApiFunction): + API_VISIBILITY = "extern %s" + + def get_wrapper(self, space): + return self.callable + + def __call__(self, *args): + raise TypeError("the function %s should not be directly " + "called from RPython, but only from C" % (self.func,)) + +def c_only(argtypes, restype): + def decorate(func): + header = DEFAULT_HEADER + if func.__name__ in FUNCTIONS_BY_HEADER[header]: + raise ValueError("%s already registered" % func.__name__) + api_function = COnlyApiFunction(argtypes, restype, func) + FUNCTIONS_BY_HEADER[header][func.__name__] = api_function + return api_function + return decorate + def api_func_from_cdef(func, cdef, cts, error=_NOT_SPECIFIED, header=DEFAULT_HEADER, result_is_ll=False): @@ -613,6 +651,11 @@ 'PyBytes_FromFormat', 'PyBytes_FromFormatV', 'PyType_FromSpec', + 'Py_IncRef', 'Py_DecRef', 'PyObject_Free', 'PyObject_GC_Del', 'PyType_GenericAlloc', + '_PyObject_New', '_PyObject_NewVar', + '_PyObject_GC_New', '_PyObject_GC_NewVar', + 'PyObject_Init', 'PyObject_InitVar', + 'PyTuple_New', ] TYPES = {} FORWARD_DECLS = [] @@ -950,8 +993,14 @@ # see "Handling of the GIL" above (careful, we don't have the GIL here) tid = rthread.get_or_make_ident() - _gil_auto = (gil_auto_workaround and cpyext_glob_tid_ptr[0] != tid) - if gil_acquire or _gil_auto: + _gil_auto = False + if gil_auto_workaround and cpyext_glob_tid_ptr[0] != tid: + # replace '-1' with the real tid, now that we have the tid + if cpyext_glob_tid_ptr[0] == -1: + cpyext_glob_tid_ptr[0] = tid + else: + _gil_auto = True + if _gil_auto or gil_acquire: if cpyext_glob_tid_ptr[0] == tid: deadlock_error(pname) rgil.acquire() @@ -1073,6 +1122,7 @@ setdefenc = rffi.llexternal('_%s_setfilesystemdefaultencoding' % prefix, [rffi.CCHARP], lltype.Void, compilation_info=eci, _nowrapper=True) + @init_function def init_types(space): from pypy.module.cpyext.typeobject import py_type_ready from pypy.module.sys.interp_encoding import getfilesystemencoding @@ -1080,7 +1130,7 @@ py_type_ready(space, get_capsule_type()) s = space.text_w(getfilesystemencoding(space)) setdefenc(rffi.str2charp(s, track_allocation=False)) # "leaks" - INIT_FUNCTIONS.append(init_types) + from pypy.module.posix.interp_posix import add_fork_hook global py_fatalerror py_fatalerror = rffi.llexternal('%s_FatalError' % prefix, @@ -1092,6 +1142,45 @@ _reinit_tls() add_fork_hook('child', reinit_tls) + +def attach_c_functions(space, eci, prefix): + state = space.fromcache(State) + state.C._Py_Dealloc = rffi.llexternal('_Py_Dealloc', + [PyObject], lltype.Void, + compilation_info=eci, + _nowrapper=True) + state.C.PyObject_Free = rffi.llexternal( + mangle_name(prefix, 'PyObject_Free'), + [rffi.VOIDP], lltype.Void, + compilation_info=eci, + _nowrapper=True) + state.C.PyType_GenericAlloc = rffi.llexternal( + mangle_name(prefix, 'PyType_GenericAlloc'), + [PyTypeObjectPtr, Py_ssize_t], PyObject, + compilation_info=eci, + _nowrapper=True) + state.C._PyPy_int_dealloc = rffi.llexternal( + '_PyPy_int_dealloc', [PyObject], lltype.Void, + compilation_info=eci, _nowrapper=True) + state.C.PyTuple_New = rffi.llexternal( + mangle_name(prefix, 'PyTuple_New'), + [Py_ssize_t], PyObject, + compilation_info=eci, + _nowrapper=True) + state.C._PyPy_tuple_dealloc = rffi.llexternal( + '_PyPy_tuple_dealloc', [PyObject], lltype.Void, + compilation_info=eci, _nowrapper=True) + _, state.C.set_marker = rffi.CExternVariable( + Py_ssize_t, '_pypy_rawrefcount_w_marker_deallocating', + eci, _nowrapper=True, c_type='Py_ssize_t') + state.C._PyPy_subtype_dealloc = rffi.llexternal( + '_PyPy_subtype_dealloc', [PyObject], lltype.Void, + compilation_info=eci, _nowrapper=True) + state.C._PyPy_object_dealloc = rffi.llexternal( + '_PyPy_object_dealloc', [PyObject], lltype.Void, + compilation_info=eci, _nowrapper=True) + + def init_function(func): INIT_FUNCTIONS.append(func) return func @@ -1160,6 +1249,7 @@ space.fromcache(State).install_dll(eci) modulename = py.path.local(eci.libraries[-1]) + attach_c_functions(space, eci, prefix) run_bootstrap_functions(space) # load the bridge, and init structure @@ -1200,7 +1290,6 @@ in_dll = ll2ctypes.get_ctypes_type(PyObject.TO).in_dll(bridge, mname) py_obj = ll2ctypes.ctypes2lltype(PyObject, ctypes.pointer(in_dll)) builder.prepare(py_obj, w_obj) - builder.attach_all(space) pypyAPI = ctypes.POINTER(ctypes.c_void_p).in_dll(bridge, 'pypyAPI') @@ -1211,6 +1300,12 @@ ll2ctypes.lltype2ctypes(func.get_llhelper(space)), ctypes.c_void_p) + # we need to call this *after* the init code above, because it might + # indirectly call some functions which are attached to pypyAPI (e.g., we + # if do tuple_attach of the prebuilt empty tuple, we need to call + # _PyPy_Malloc) + builder.attach_all(space) + setup_init_functions(eci, prefix) return modulename.new(ext='') @@ -1407,6 +1502,7 @@ source_dir / "pylifecycle.c", source_dir / "object.c", source_dir / "typeobject.c", + source_dir / "tupleobject.c", ] def build_eci(code, use_micronumpy=False, translating=False): @@ -1504,6 +1600,7 @@ eci = build_eci(code, use_micronumpy, translating=True) space.fromcache(State).install_dll(eci) + attach_c_functions(space, eci, prefix) run_bootstrap_functions(space) # emit uninitialized static data @@ -1703,7 +1800,7 @@ @specialize.memo() def make_generic_cpy_call(FT, expect_null, convert_result): - from pypy.module.cpyext.pyobject import is_pyobj, as_pyobj + from pypy.module.cpyext.pyobject import is_pyobj, make_ref, decref from pypy.module.cpyext.pyobject import get_w_obj_and_decref from pypy.module.cpyext.pyerrors import PyErr_Occurred unrolling_arg_types = unrolling_iterable(enumerate(FT.ARGS)) @@ -1731,15 +1828,17 @@ @specialize.ll() def generic_cpy_call(space, func, *args): boxed_args = () - keepalives = () + to_decref = () assert len(args) == len(FT.ARGS) for i, ARG in unrolling_arg_types: arg = args[i] + _pyobj = None if is_PyObject(ARG): if not is_pyobj(arg): - keepalives += (arg,) - arg = as_pyobj(space, arg) + arg = make_ref(space, arg) + _pyobj = arg boxed_args += (arg,) + to_decref += (_pyobj,) # see "Handling of the GIL" above tid = rthread.get_ident() @@ -1753,7 +1852,11 @@ finally: assert cpyext_glob_tid_ptr[0] == tid cpyext_glob_tid_ptr[0] = 0 - keepalive_until_here(*keepalives) + for i, ARG in unrolling_arg_types: + # note that this loop is nicely unrolled statically by RPython + _pyobj = to_decref[i] + if _pyobj is not None: + decref(space, _pyobj) if convert_result and is_PyObject(RESULT_TYPE): if not is_pyobj(result): diff --git a/pypy/module/cpyext/bytearrayobject.py b/pypy/module/cpyext/bytearrayobject.py --- a/pypy/module/cpyext/bytearrayobject.py +++ b/pypy/module/cpyext/bytearrayobject.py @@ -3,12 +3,12 @@ from pypy.interpreter.error import OperationError, oefmt from pypy.objspace.std.bytearrayobject import new_bytearray from pypy.module.cpyext.api import ( - cpython_api, cpython_struct, bootstrap_function, build_type_checkers, - PyVarObjectFields, Py_ssize_t, CONST_STRING, CANNOT_FAIL) + cpython_api, cpython_struct, build_type_checkers, + PyVarObjectFields, Py_ssize_t, CONST_STRING) from pypy.module.cpyext.pyerrors import PyErr_BadArgument from pypy.module.cpyext.pyobject import ( - PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, - make_typedescr, get_typedescr, Py_IncRef) + PyObject, PyObjectP, make_ref, from_ref, + make_typedescr, get_typedescr) # Type PyByteArrayObject represents a mutable array of bytes. # The Python API is that of a sequence; # the bytes are mapped to ints in [0, 256). diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py --- a/pypy/module/cpyext/bytesobject.py +++ b/pypy/module/cpyext/bytesobject.py @@ -5,8 +5,8 @@ PyVarObjectFields, Py_ssize_t, CONST_STRING, CANNOT_FAIL, slot_function) from pypy.module.cpyext.pyerrors import PyErr_BadArgument from pypy.module.cpyext.pyobject import ( - PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference, - make_typedescr, get_typedescr, as_pyobj, Py_IncRef, get_w_obj_and_decref, + PyObject, PyObjectP, decref, make_ref, from_ref, track_reference, + make_typedescr, get_typedescr, as_pyobj, get_w_obj_and_decref, pyobj_has_w_obj) from pypy.objspace.std.bytesobject import W_BytesObject @@ -203,7 +203,7 @@ try: py_newstr = new_empty_str(space, newsize) except MemoryError: - Py_DecRef(space, ref[0]) + decref(space, ref[0]) ref[0] = lltype.nullptr(PyObject.TO) raise to_cp = newsize @@ -212,7 +212,7 @@ to_cp = oldsize for i in range(to_cp): py_newstr.c_ob_sval[i] = py_str.c_ob_sval[i] - Py_DecRef(space, ref[0]) + decref(space, ref[0]) ref[0] = rffi.cast(PyObject, py_newstr) return 0 @@ -246,7 +246,7 @@ try: PyBytes_Concat(space, ref, newpart) finally: - Py_DecRef(space, newpart) + decref(space, newpart) @cpython_api([PyObject, PyObject], PyObject) def _PyBytes_Join(space, w_sep, w_seq): diff --git a/pypy/module/cpyext/classobject.py b/pypy/module/cpyext/classobject.py --- a/pypy/module/cpyext/classobject.py +++ b/pypy/module/cpyext/classobject.py @@ -1,5 +1,5 @@ from rpython.rtyper.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL +from pypy.module.cpyext.api import CANNOT_FAIL, cpython_api from pypy.module.cpyext.pyobject import PyObject from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.function import Method diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py --- a/pypy/module/cpyext/dictobject.py +++ b/pypy/module/cpyext/dictobject.py @@ -9,7 +9,7 @@ bootstrap_function, slot_function) from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, as_pyobj, make_typedescr, track_reference, create_ref, from_ref, decref, - Py_IncRef) + incref) from pypy.module.cpyext.object import _dealloc from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall @@ -279,7 +279,7 @@ w_keys = space.newlist(space.listview(w_keyview)) w_keys.switch_to_object_strategy() py_dict.c__tmpkeys = create_ref(space, w_keys) - Py_IncRef(space, py_dict.c__tmpkeys) + incref(space, py_dict.c__tmpkeys) else: if not py_dict.c__tmpkeys: # pos should have been 0, cannot fail so return 0 diff --git a/pypy/module/cpyext/exception.py b/pypy/module/cpyext/exception.py --- a/pypy/module/cpyext/exception.py +++ b/pypy/module/cpyext/exception.py @@ -1,7 +1,7 @@ # Provide implementation of PyException_ functions. from pypy.module.cpyext.api import cpython_api -from pypy.module.cpyext.pyobject import PyObject, from_ref, Py_DecRef +from pypy.module.cpyext.pyobject import PyObject, from_ref, decref from rpython.rtyper.lltypesystem import rffi, lltype @@ -48,7 +48,7 @@ This steals a reference to ctx.""" if ctx: w_ctx = from_ref(space, ctx) - Py_DecRef(space, ctx) + decref(space, ctx) else: w_ctx = space.w_None space.setattr(w_exc, space.newtext('__context__'), w_ctx) @@ -72,7 +72,7 @@ This steals a reference to cause.""" if cause: w_cause = from_ref(space, cause) - Py_DecRef(space, cause) + decref(space, cause) else: w_cause = space.w_None space.setattr(w_exc, space.newtext('__cause__'), w_cause) diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py --- a/pypy/module/cpyext/frameobject.py +++ b/pypy/module/cpyext/frameobject.py @@ -3,7 +3,7 @@ cpython_api, bootstrap_function, PyObjectFields, cpython_struct, CANNOT_FAIL, slot_function) from pypy.module.cpyext.pyobject import ( - PyObject, Py_DecRef, make_ref, from_ref, track_reference, + PyObject, decref, make_ref, from_ref, track_reference, make_typedescr, get_typedescr) from pypy.module.cpyext.state import State from pypy.module.cpyext.pystate import PyThreadState @@ -43,9 +43,9 @@ def frame_dealloc(space, py_obj): py_frame = rffi.cast(PyFrameObject, py_obj) py_code = rffi.cast(PyObject, py_frame.c_f_code) - Py_DecRef(space, py_code) - Py_DecRef(space, py_frame.c_f_globals) - Py_DecRef(space, py_frame.c_f_locals) + decref(space, py_code) + decref(space, py_frame.c_f_globals) + decref(space, py_frame.c_f_locals) from pypy.module.cpyext.object import _dealloc _dealloc(space, py_obj) diff --git a/pypy/module/cpyext/funcobject.py b/pypy/module/cpyext/funcobject.py --- a/pypy/module/cpyext/funcobject.py +++ b/pypy/module/cpyext/funcobject.py @@ -4,7 +4,7 @@ cpython_api, bootstrap_function, cpython_struct, build_type_checkers, slot_function) from pypy.module.cpyext.pyobject import ( - PyObject, make_ref, from_ref, Py_DecRef, make_typedescr) + PyObject, make_ref, from_ref, decref, make_typedescr) from rpython.rlib.unroll import unrolling_iterable from pypy.interpreter.error import OperationError from pypy.interpreter.function import Function, Method @@ -62,7 +62,7 @@ @slot_function([PyObject], lltype.Void) def function_dealloc(space, py_obj): py_func = rffi.cast(PyFunctionObject, py_obj) - Py_DecRef(space, py_func.c_func_name) + decref(space, py_func.c_func_name) from pypy.module.cpyext.object import _dealloc _dealloc(space, py_obj) @@ -81,8 +81,8 @@ @slot_function([PyObject], lltype.Void) def code_dealloc(space, py_obj): py_code = rffi.cast(PyCodeObject, py_obj) - Py_DecRef(space, py_code.c_co_name) - Py_DecRef(space, py_code.c_co_filename) + decref(space, py_code.c_co_name) + decref(space, py_code.c_co_filename) from pypy.module.cpyext.object import _dealloc _dealloc(space, py_obj) diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h --- a/pypy/module/cpyext/include/object.h +++ b/pypy/module/cpyext/include/object.h @@ -59,6 +59,11 @@ } while (0) #endif +PyAPI_FUNC(void) Py_IncRef(PyObject *); +PyAPI_FUNC(void) Py_DecRef(PyObject *); +extern Py_ssize_t _pypy_rawrefcount_w_marker_deallocating; +PyAPI_FUNC(void) _Py_Dealloc(PyObject *); + #define Py_CLEAR(op) \ do { \ @@ -87,6 +92,10 @@ #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) #define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) +#define _Py_NewReference(op) \ + ( ((PyObject *)(op))->ob_refcnt = 1, \ + ((PyObject *)(op))->ob_pypy_link = 0 ) + #define _Py_ForgetReference(ob) /* nothing */ #define Py_None (&_Py_NoneStruct) @@ -246,8 +255,16 @@ ) & ~(SIZEOF_VOID_P - 1) \ ) -#define PyObject_INIT PyObject_Init -#define PyObject_INIT_VAR PyObject_InitVar + +#define PyObject_INIT(op, typeobj) \ + ( Py_TYPE(op) = (typeobj), ((PyObject *)(op))->ob_refcnt = 1,\ + ((PyObject *)(op))->ob_pypy_link = 0, (op) ) +#define PyObject_INIT_VAR(op, typeobj, size) \ + ( Py_SIZE(op) = (size), PyObject_INIT((op), (typeobj)) ) + + +PyAPI_FUNC(PyObject *) PyType_GenericAlloc(PyTypeObject *, Py_ssize_t); + /* #define PyObject_NEW(type, typeobj) \ ( (type *) PyObject_Init( \ @@ -334,12 +351,26 @@ */ +/* on CPython, these are in objimpl.h */ + +PyAPI_FUNC(void) PyObject_Free(void *); +PyAPI_FUNC(void) PyObject_GC_Del(void *); + #define PyObject_MALLOC PyObject_Malloc #define PyObject_REALLOC PyObject_Realloc #define PyObject_FREE PyObject_Free #define PyObject_Del PyObject_Free #define PyObject_DEL PyObject_Free +PyAPI_FUNC(PyObject *) _PyObject_New(PyTypeObject *); +PyAPI_FUNC(PyVarObject *) _PyObject_NewVar(PyTypeObject *, Py_ssize_t); +PyAPI_FUNC(PyObject *) _PyObject_GC_New(PyTypeObject *); +PyAPI_FUNC(PyVarObject *) _PyObject_GC_NewVar(PyTypeObject *, Py_ssize_t); + +PyAPI_FUNC(PyObject *) PyObject_Init(PyObject *, PyTypeObject *); +PyAPI_FUNC(PyVarObject *) PyObject_InitVar(PyVarObject *, + PyTypeObject *, Py_ssize_t); + #ifndef Py_LIMITED_API PyAPI_FUNC(int) PyObject_CallFinalizerFromDealloc(PyObject *); #endif @@ -349,6 +380,8 @@ PyAPI_FUNC(int) PyPyType_Register(PyTypeObject *); #define PyObject_Length PyObject_Size #define _PyObject_GC_Del PyObject_GC_Del +PyAPI_FUNC(void) _PyPy_subtype_dealloc(PyObject *); +PyAPI_FUNC(void) _PyPy_object_dealloc(PyObject *); #ifdef __cplusplus diff --git a/pypy/module/cpyext/include/tupleobject.h b/pypy/module/cpyext/include/tupleobject.h --- a/pypy/module/cpyext/include/tupleobject.h +++ b/pypy/module/cpyext/include/tupleobject.h @@ -16,9 +16,13 @@ */ } PyTupleObject; +PyAPI_FUNC(PyObject *) PyTuple_New(Py_ssize_t size); +PyAPI_FUNC(void) _PyPy_tuple_dealloc(PyObject *); + /* defined in varargswrapper.c */ PyAPI_FUNC(PyObject *) PyTuple_Pack(Py_ssize_t, ...); + /* Macro, trading safety for speed */ #define PyTuple_GET_ITEM(op, i) (((PyTupleObject *)(op))->ob_item[i]) #define PyTuple_GET_SIZE(op) Py_SIZE(op) diff --git a/pypy/module/cpyext/mapping.py b/pypy/module/cpyext/mapping.py --- a/pypy/module/cpyext/mapping.py +++ b/pypy/module/cpyext/mapping.py @@ -1,7 +1,8 @@ from rpython.rtyper.lltypesystem import lltype, rffi from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, CONST_STRING, Py_ssize_t) -from pypy.module.cpyext.pyobject import PyObject +from pypy.module.cpyext.pyobject import ( + PyObject, hack_for_result_often_existing_obj) @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) @@ -40,12 +41,13 @@ return space.call_function(space.w_list, space.call_method(w_obj, "items")) - at cpython_api([PyObject, CONST_STRING], PyObject) + at cpython_api([PyObject, CONST_STRING], PyObject, result_is_ll=True) def PyMapping_GetItemString(space, w_obj, key): """Return element of o corresponding to the object key or NULL on failure. This is the equivalent of the Python expression o[key].""" w_key = space.newtext(rffi.charp2str(key)) - return space.getitem(w_obj, w_key) + w_res = space.getitem(w_obj, w_key) + return hack_for_result_often_existing_obj(space, w_res) @cpython_api([PyObject, CONST_STRING, PyObject], rffi.INT_real, error=-1) def PyMapping_SetItemString(space, w_obj, key, w_value): diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -14,7 +14,9 @@ cpython_api, generic_cpy_call, CANNOT_FAIL, slot_function, cts, build_type_checkers) from pypy.module.cpyext.pyobject import ( - Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr) + decref, from_ref, make_ref, as_pyobj, make_typedescr) +from pypy.module.cpyext.state import State +from pypy.module.cpyext.tupleobject import tuple_from_args_w PyMethodDef = cts.gettype('PyMethodDef') PyCFunction = cts.gettype('PyCFunction') @@ -38,8 +40,8 @@ @slot_function([PyObject], lltype.Void) def cfunction_dealloc(space, py_obj): py_func = rffi.cast(PyCFunctionObject, py_obj) - Py_DecRef(space, py_func.c_m_self) - Py_DecRef(space, py_func.c_m_module) + decref(space, py_func.c_m_self) + decref(space, py_func.c_m_module) from pypy.module.cpyext.object import _dealloc _dealloc(space, py_obj) @@ -77,31 +79,29 @@ return None class W_PyCFunctionObject(W_Root): - # TODO create a slightly different class depending on the c_ml_flags + _immutable_fields_ = ["flags"] + def __init__(self, space, ml, w_self, w_module=None): self.ml = ml self.name = rffi.charp2str(rffi.cast(rffi.CCHARP, self.ml.c_ml_name)) + self.flags = rffi.cast(lltype.Signed, self.ml.c_ml_flags) self.w_self = w_self self.w_module = w_module - def call(self, space, w_self, w_args, w_kw): - # Call the C function - if w_self is None: - w_self = self.w_self - flags = rffi.cast(lltype.Signed, self.ml.c_ml_flags) - flags &= ~(METH_CLASS | METH_STATIC | METH_COEXIST) - if not flags & METH_KEYWORDS and space.is_true(w_kw): + def descr_call(self, space, __args__): + return self.call(space, self.w_self, __args__) + + def call(self, space, w_self, __args__): + flags = self.flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST) + length = len(__args__.arguments_w) + if not flags & METH_KEYWORDS and __args__.keywords: raise oefmt(space.w_TypeError, "%s() takes no keyword arguments", self.name) - - func = self.ml.c_ml_meth - length = space.int_w(space.len(w_args)) if flags & METH_KEYWORDS: - func = rffi.cast(PyCFunctionKwArgs, self.ml.c_ml_meth) - return generic_cpy_call(space, func, w_self, w_args, w_kw) + return self.call_keywords(space, w_self, __args__) elif flags & METH_NOARGS: if length == 0: - return generic_cpy_call(space, func, w_self, None) + return self.call_noargs(space, w_self, __args__) raise oefmt(space.w_TypeError, "%s() takes no arguments", self.name) elif flags & METH_O: @@ -109,13 +109,47 @@ raise oefmt(space.w_TypeError, "%s() takes exactly one argument (%d given)", self.name, length) - w_arg = space.getitem(w_args, space.newint(0)) - return generic_cpy_call(space, func, w_self, w_arg) + return self.call_o(space, w_self, __args__) elif flags & METH_VARARGS: - return generic_cpy_call(space, func, w_self, w_args) + return self.call_varargs(space, w_self, __args__) else: # shouldn't happen! raise oefmt(space.w_RuntimeError, "unknown calling convention") + def call_noargs(self, space, w_self, __args__): + func = self.ml.c_ml_meth + return generic_cpy_call(space, func, w_self, None) + + def call_o(self, space, w_self, __args__): + func = self.ml.c_ml_meth + w_o = __args__.arguments_w[0] + return generic_cpy_call(space, func, w_self, w_o) + + def call_varargs(self, space, w_self, __args__): + state = space.fromcache(State) + func = self.ml.c_ml_meth + py_args = tuple_from_args_w(space, __args__.arguments_w) + try: + return generic_cpy_call(space, func, w_self, py_args) + finally: + decref(space, py_args) + + def call_keywords(self, space, w_self, __args__): + func = rffi.cast(PyCFunctionKwArgs, self.ml.c_ml_meth) + py_args = tuple_from_args_w(space, __args__.arguments_w) + w_kwargs = None + if __args__.keywords: + # CCC: we should probably have a @jit.look_inside_iff if the + # keyword count is constant, as we do in Arguments.unpack + w_kwargs = space.newdict() + for i in range(len(__args__.keywords)): + key = __args__.keywords[i] + w_obj = __args__.keywords_w[i] + space.setitem(w_kwargs, space.newtext(key), w_obj) + try: + return generic_cpy_call(space, func, w_self, py_args, w_kwargs) + finally: + decref(space, py_args) + def get_doc(self, space): c_doc = self.ml.c_ml_doc if c_doc: @@ -134,27 +168,11 @@ return space.newtext(txtsig) return space.w_None -class W_PyCFunctionObjectNoArgs(W_PyCFunctionObject): - def call(self, space, w_self, w_args, w_kw): - # Call the C function - if w_self is None: - w_self = self.w_self - func = self.ml.c_ml_meth - return generic_cpy_call(space, func, w_self, None) +class W_PyCMethodObject(W_PyCFunctionObject): -class W_PyCFunctionObjectSingleObject(W_PyCFunctionObject): - def call(self, space, w_self, w_o, w_kw): - if w_self is None: - w_self = self.w_self - func = self.ml.c_ml_meth - return generic_cpy_call(space, func, w_self, w_o) - -class W_PyCMethodObject(W_PyCFunctionObject): - w_self = None def __init__(self, space, ml, w_type): + W_PyCFunctionObject.__init__(self, space, ml, w_self=None) self.space = space - self.ml = ml - self.name = rffi.charp2str(rffi.cast(rffi.CCHARP, ml.c_ml_name)) self.w_objclass = w_type def __repr__(self): @@ -167,14 +185,13 @@ self.name, w_objclass.name)) def descr_call(self, space, __args__): - args_w, kw_w = __args__.unpack() - if len(args_w) < 1: + if len(__args__.arguments_w) == 0: w_objclass = self.w_objclass assert isinstance(w_objclass, W_TypeObject) raise oefmt(space.w_TypeError, "descriptor '%8' of '%s' object needs an argument", self.name, w_objclass.name) - w_instance = args_w[0] + w_instance = __args__.arguments_w[0] # XXX: needs a stricter test if not space.isinstance_w(w_instance, self.w_objclass): w_objclass = self.w_objclass @@ -182,12 +199,10 @@ raise oefmt(space.w_TypeError, "descriptor '%8' requires a '%s' object but received a '%T'", self.name, w_objclass.name, w_instance) - w_args = space.newtuple(args_w[1:]) - w_kw = space.newdict() - for key, w_obj in kw_w.items(): - space.setitem(w_kw, space.newtext(key), w_obj) - ret = self.call(space, w_instance, w_args, w_kw) - return ret + # + # CCC: we can surely do better than this + __args__ = __args__.replace_arguments(__args__.arguments_w[1:]) + return self.call(space, w_instance, __args__) # PyPy addition, for Cython _, _ = build_type_checkers("MethodDescr", W_PyCMethodObject) @@ -203,16 +218,25 @@ return isinstance(w_obj, BuiltinFunction) class W_PyCClassMethodObject(W_PyCFunctionObject): - w_self = None + def __init__(self, space, ml, w_type): + W_PyCFunctionObject.__init__(self, space, ml, w_self=None) self.space = space - self.ml = ml - self.name = rffi.charp2str(rffi.cast(rffi.CCHARP, ml.c_ml_name)) self.w_objclass = w_type def __repr__(self): return self.space.unwrap(self.descr_method_repr()) + def descr_call(self, space, __args__): + if len(__args__.arguments_w) == 0: + raise oefmt(space.w_TypeError, + "descriptor '%s' of '%s' object needs an argument", + self.name, self.w_objclass.getname(space)) + w_instance = __args__.arguments_w[0] # XXX typecheck missing + # CCC: we can surely do better than this + __args__ = __args__.replace_arguments(__args__.arguments_w[1:]) + return self.call(space, w_instance, __args__) + def descr_method_repr(self): return self.getrepr( self.space, u"built-in method '%s' of '%s' object" % @@ -278,45 +302,6 @@ space.setitem(w_kw, space.newtext(key), w_obj) return self.call(space, w_self, w_args, w_kw) -def cfunction_descr_call_noargs(space, w_self): - # special case for calling with flags METH_NOARGS - self = space.interp_w(W_PyCFunctionObjectNoArgs, w_self) - return self.call(space, None, None, None) - -def cfunction_descr_call_single_object(space, w_self, w_o): - # special case for calling with flags METH_O - self = space.interp_w(W_PyCFunctionObjectSingleObject, w_self) - return self.call(space, None, w_o, None) - - at jit.dont_look_inside -def cfunction_descr_call(space, w_self, __args__): - # specialize depending on the W_PyCFunctionObject - self = space.interp_w(W_PyCFunctionObject, w_self) - args_w, kw_w = __args__.unpack() - # XXX __args__.unpack is slow - w_args = space.newtuple(args_w) - w_kw = space.newdict() - for key, w_obj in kw_w.items(): - space.setitem(w_kw, space.newtext(key), w_obj) - ret = self.call(space, None, w_args, w_kw) - return ret - - at jit.dont_look_inside -def cclassmethod_descr_call(space, w_self, __args__): - self = space.interp_w(W_PyCFunctionObject, w_self) - args_w, kw_w = __args__.unpack() - if len(args_w) < 1: - raise oefmt(space.w_TypeError, - "descriptor '%8' of '%N' object needs an argument", - self.name, self.w_objclass) - w_instance = args_w[0] # XXX typecheck missing - w_args = space.newtuple(args_w[1:]) - w_kw = space.newdict() - for key, w_obj in kw_w.items(): - space.setitem(w_kw, space.newtext(key), w_obj) - ret = self.call(space, w_instance, w_args, w_kw) - return ret - def cmethod_descr_get(space, w_function, w_obj, w_cls=None): if w_obj is None or space.is_w(w_obj, space.w_None): return w_function @@ -331,7 +316,7 @@ W_PyCFunctionObject.typedef = TypeDef( 'builtin_function_or_method', - __call__ = interp2app(cfunction_descr_call), + __call__ = interp2app(W_PyCFunctionObject.descr_call), __doc__ = GetSetProperty(W_PyCFunctionObject.get_doc), __text_signature__ = GetSetProperty(W_PyCFunctionObject.get_txtsig), __module__ = interp_attrproperty_w('w_module', cls=W_PyCFunctionObject), @@ -340,28 +325,6 @@ ) W_PyCFunctionObject.typedef.acceptable_as_base_class = False -W_PyCFunctionObjectNoArgs.typedef = TypeDef( - 'builtin_function_or_method', W_PyCFunctionObject.typedef, - __call__ = interp2app(cfunction_descr_call_noargs), - __doc__ = GetSetProperty(W_PyCFunctionObjectNoArgs.get_doc), - __text_signature__ = GetSetProperty(W_PyCFunctionObjectNoArgs.get_txtsig), - __module__ = interp_attrproperty_w('w_module', cls=W_PyCFunctionObjectNoArgs), - __name__ = interp_attrproperty('name', cls=W_PyCFunctionObjectNoArgs, - wrapfn="newtext_or_none"), - ) -W_PyCFunctionObjectNoArgs.typedef.acceptable_as_base_class = False - -W_PyCFunctionObjectSingleObject.typedef = TypeDef( - 'builtin_function_or_method', W_PyCFunctionObject.typedef, - __call__ = interp2app(cfunction_descr_call_single_object), - __doc__ = GetSetProperty(W_PyCFunctionObjectSingleObject.get_doc), - __text_signature__ = GetSetProperty(W_PyCFunctionObjectSingleObject.get_txtsig), - __module__ = interp_attrproperty_w('w_module', cls=W_PyCFunctionObjectSingleObject), - __name__ = interp_attrproperty('name', cls=W_PyCFunctionObjectSingleObject, - wrapfn="newtext_or_none"), - ) -W_PyCFunctionObjectSingleObject.typedef.acceptable_as_base_class = False - W_PyCMethodObject.typedef = TypeDef( 'method_descriptor', __get__ = interp2app(cmethod_descr_get), @@ -376,7 +339,7 @@ W_PyCClassMethodObject.typedef = TypeDef( 'classmethod', __get__ = interp2app(cclassmethod_descr_get), - __call__ = interp2app(cclassmethod_descr_call), + __call__ = interp2app(W_PyCClassMethodObject.descr_call), __name__ = interp_attrproperty('name', cls=W_PyCClassMethodObject, wrapfn="newtext_or_none"), __objclass__ = interp_attrproperty_w('w_objclass', diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py --- a/pypy/module/cpyext/modsupport.py +++ b/pypy/module/cpyext/modsupport.py @@ -8,8 +8,7 @@ from pypy.interpreter.module import Module from pypy.module.cpyext.methodobject import ( W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod, - PyMethodDef, PyDescr_NewClassMethod, PyStaticMethod_New, - W_PyCFunctionObjectNoArgs, W_PyCFunctionObjectSingleObject) + PyMethodDef, PyDescr_NewClassMethod, PyStaticMethod_New) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.module.cpyext.state import State from pypy.interpreter.error import oefmt @@ -167,14 +166,6 @@ "exception", w_mod.w_name) cur_slot = rffi.ptradd(cur_slot, 1) -def _create_pyc_function_object(space, method, w_self, w_name, flags): - flags &= ~(METH_CLASS | METH_STATIC | METH_COEXIST) - if flags == METH_NOARGS: - return W_PyCFunctionObjectNoArgs(space, method, w_self, w_name) - if flags == METH_O: - return W_PyCFunctionObjectSingleObject(space, method, w_self, w_name) - return W_PyCFunctionObject(space, method, w_self, w_name) - def convert_method_defs(space, dict_w, methods, w_type, w_self=None, name=None): w_name = space.newtext_or_none(name) methods = rffi.cast(rffi.CArrayPtr(PyMethodDef), methods) @@ -193,8 +184,7 @@ raise oefmt(space.w_ValueError, "module functions cannot set METH_CLASS or " "METH_STATIC") - w_obj = _create_pyc_function_object(space, method, w_self, - w_name, flags) + w_obj = W_PyCFunctionObject(space, method, w_self, w_name) else: if methodname in dict_w and not (flags & METH_COEXIST): continue diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py --- a/pypy/module/cpyext/object.py +++ b/pypy/module/cpyext/object.py @@ -3,9 +3,10 @@ cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, PyVarObject, size_t, slot_function, cts, Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT, - Py_GE, CONST_STRING, FILEP, fwrite) + Py_GE, CONST_STRING, FILEP, fwrite, c_only) from pypy.module.cpyext.pyobject import ( - PyObject, PyObjectP, from_ref, Py_IncRef, Py_DecRef, get_typedescr) + PyObject, PyObjectP, from_ref, incref, decref, + get_typedescr, hack_for_result_often_existing_obj) from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall from pypy.objspace.std.typeobject import W_TypeObject @@ -32,31 +33,20 @@ # XXX FIXME return realloc(ptr, size) - at cpython_api([rffi.VOIDP], lltype.Void) -def PyObject_Free(space, ptr): + at c_only([rffi.VOIDP], lltype.Void) +def _PyPy_Free(ptr): lltype.free(ptr, flavor='raw') - at cpython_api([PyTypeObjectPtr], PyObject, result_is_ll=True) -def _PyObject_New(space, type): - return _PyObject_NewVar(space, type, 0) + at c_only([Py_ssize_t], rffi.VOIDP) +def _PyPy_Malloc(size): + # XXX: the malloc inside BaseCpyTypedescr.allocate and + # typeobject.type_alloc specify zero=True, so this is why we use it also + # here. However, CPython does a simple non-initialized malloc, so we + # should investigate whether we can remove zero=True as well + return lltype.malloc(rffi.VOIDP.TO, size, + flavor='raw', zero=True, + add_memory_pressure=True) - at cpython_api([PyTypeObjectPtr, Py_ssize_t], PyObject, result_is_ll=True) -def _PyObject_NewVar(space, type, itemcount): - w_type = from_ref(space, rffi.cast(PyObject, type)) - assert isinstance(w_type, W_TypeObject) - typedescr = get_typedescr(w_type.layout.typedef) - py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) - #py_obj.c_ob_refcnt = 0 --- will be set to 1 again by PyObject_Init{Var} - if type.c_tp_itemsize == 0: - w_obj = PyObject_Init(space, py_obj, type) - else: - py_objvar = rffi.cast(PyVarObject, py_obj) - w_obj = PyObject_InitVar(space, py_objvar, type, itemcount) - return py_obj - - at slot_function([PyObject], lltype.Void) -def PyObject_dealloc(space, obj): - return _dealloc(space, obj) def _dealloc(space, obj): # This frees an object after its refcount dropped to zero, so we @@ -66,19 +56,7 @@ obj_voidp = rffi.cast(rffi.VOIDP, obj) generic_cpy_call(space, pto.c_tp_free, obj_voidp) if pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE: - Py_DecRef(space, rffi.cast(PyObject, pto)) - - at cpython_api([PyTypeObjectPtr], PyObject, result_is_ll=True) -def _PyObject_GC_New(space, type): - return _PyObject_New(space, type) - - at cpython_api([PyTypeObjectPtr, Py_ssize_t], PyObject, result_is_ll=True) -def _PyObject_GC_NewVar(space, type, itemcount): - return _PyObject_NewVar(space, type, itemcount) - - at cpython_api([rffi.VOIDP], lltype.Void) -def PyObject_GC_Del(space, obj): - PyObject_Free(space, obj) + decref(space, rffi.cast(PyObject, pto)) @cpython_api([PyObject], PyObjectP, error=CANNOT_FAIL) def _PyObject_GetDictPtr(space, op): @@ -92,20 +70,22 @@ def PyObject_Not(space, w_obj): return not space.is_true(w_obj) - at cpython_api([PyObject, PyObject], PyObject) + at cpython_api([PyObject, PyObject], PyObject, result_is_ll=True) def PyObject_GetAttr(space, w_obj, w_name): """Retrieve an attribute named attr_name from object o. Returns the attribute value on success, or NULL on failure. This is the equivalent of the Python expression o.attr_name.""" - return space.getattr(w_obj, w_name) + w_res = space.getattr(w_obj, w_name) + return hack_for_result_often_existing_obj(space, w_res) - at cpython_api([PyObject, CONST_STRING], PyObject) + at cpython_api([PyObject, CONST_STRING], PyObject, result_is_ll=True) def PyObject_GetAttrString(space, w_obj, name_ptr): """Retrieve an attribute named attr_name from object o. Returns the attribute value on success, or NULL on failure. This is the equivalent of the Python expression o.attr_name.""" name = rffi.charp2str(name_ptr) - return space.getattr(w_obj, space.newtext(name)) + w_res = space.getattr(w_obj, space.newtext(name)) + return hack_for_result_often_existing_obj(space, w_res) @cpython_api([PyObject, PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyObject_HasAttr(space, w_obj, w_name): @@ -164,11 +144,12 @@ and 0 otherwise. This function always succeeds.""" return int(space.is_true(space.callable(w_obj))) - at cpython_api([PyObject, PyObject], PyObject) + at cpython_api([PyObject, PyObject], PyObject, result_is_ll=True) def PyObject_GetItem(space, w_obj, w_key): """Return element of o corresponding to the object key or NULL on failure. This is the equivalent of the Python expression o[key].""" - return space.getitem(w_obj, w_key) + w_res = space.getitem(w_obj, w_key) + return hack_for_result_often_existing_obj(space, w_res) @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) def PyObject_SetItem(space, w_obj, w_key, w_value): @@ -184,29 +165,6 @@ space.delitem(w_obj, w_key) return 0 - at cpython_api([PyObject, PyTypeObjectPtr], PyObject, result_is_ll=True) -def PyObject_Init(space, obj, type): - """Initialize a newly-allocated object op with its type and initial - reference. Returns the initialized object. If type indicates that the - object participates in the cyclic garbage detector, it is added to the - detector's set of observed objects. Other fields of the object are not - affected.""" - if not obj: - PyErr_NoMemory(space) - obj.c_ob_type = type - obj.c_ob_pypy_link = 0 - obj.c_ob_refcnt = 1 - return obj - - at cpython_api([PyVarObject, PyTypeObjectPtr, Py_ssize_t], PyObject, result_is_ll=True) -def PyObject_InitVar(space, py_obj, type, size): - """This does everything PyObject_Init() does, and also initializes the - length information for a variable-size object.""" - if not py_obj: - PyErr_NoMemory(space) - py_obj.c_ob_size = size - return PyObject_Init(space, rffi.cast(PyObject, py_obj), type) - @cpython_api([PyObject], PyObject) def PyObject_Type(space, w_obj): """When o is non-NULL, returns a type object corresponding to the object type @@ -315,7 +273,7 @@ @cpython_api([PyObject], PyObject, result_is_ll=True) def PyObject_SelfIter(space, ref): """Undocumented function, this is what CPython does.""" - Py_IncRef(space, ref) + incref(space, ref) return ref @cpython_api([PyObject, PyObject], PyObject) diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py --- a/pypy/module/cpyext/pyerrors.py +++ b/pypy/module/cpyext/pyerrors.py @@ -10,7 +10,7 @@ from pypy.module.exceptions.interp_exceptions import W_RuntimeWarning from pypy.module.exceptions.interp_exceptions import W_StopIteration from pypy.module.cpyext.pyobject import ( - PyObject, PyObjectP, make_ref, from_ref, Py_DecRef) + PyObject, PyObjectP, make_ref, from_ref, decref, get_w_obj_and_decref) from pypy.module.cpyext.state import State from pypy.module.cpyext.import_ import PyImport_Import from rpython.rlib import rposix, jit @@ -40,7 +40,7 @@ @slot_function([PyObject], lltype.Void) def stopiteration_dealloc(space, py_obj): py_stopiteration = rffi.cast(PyStopIterationObject, py_obj) - Py_DecRef(space, py_stopiteration.c_value) + decref(space, py_stopiteration.c_value) from pypy.module.cpyext.object import _dealloc _dealloc(space, py_obj) @@ -97,7 +97,7 @@ ptraceback[0] = lltype.nullptr(PyObject.TO) @cpython_api([PyObject, PyObject, PyObject], lltype.Void) -def PyErr_Restore(space, w_type, w_value, w_traceback): +def PyErr_Restore(space, py_type, py_value, py_traceback): """Set the error indicator from the three objects. If the error indicator is already set, it is cleared first. If the objects are NULL, the error indicator is cleared. Do not pass a NULL type and non-NULL value or @@ -112,13 +112,14 @@ error indicator temporarily; use PyErr_Fetch() to save the current exception state.""" state = space.fromcache(State) + w_type = get_w_obj_and_decref(space, py_type) + w_value = get_w_obj_and_decref(space, py_value) + w_traceback = get_w_obj_and_decref(space, py_traceback) + # XXX do something with w_traceback if w_type is None: state.clear_exception() return state.set_exception(OperationError(w_type, w_value)) - Py_DecRef(space, w_type) - Py_DecRef(space, w_value) - Py_DecRef(space, w_traceback) @cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void) def PyErr_NormalizeException(space, exc_p, val_p, tb_p): @@ -140,8 +141,8 @@ w_evalue = space.w_None operr = OperationError(w_etype, w_evalue) operr.normalize_exception(space) - Py_DecRef(space, exc_p[0]) - Py_DecRef(space, val_p[0]) + decref(space, exc_p[0]) + decref(space, val_p[0]) exc_p[0] = make_ref(space, operr.w_type) val_p[0] = make_ref(space, operr.get_w_value(space)) @@ -424,7 +425,7 @@ ptraceback[0] = lltype.nullptr(PyObject.TO) @cpython_api([PyObject, PyObject, PyObject], lltype.Void) -def PyErr_SetExcInfo(space, w_type, w_value, w_traceback): +def PyErr_SetExcInfo(space, py_type, py_value, py_traceback): """---Cython extension--- Set the exception info, as known from ``sys.exc_info()``. This refers @@ -440,6 +441,9 @@ restore the exception state temporarily. Use :c:func:`PyErr_GetExcInfo` to read the exception state. """ + w_type = get_w_obj_and_decref(space, py_type) + w_value = get_w_obj_and_decref(space, py_value) + w_traceback = get_w_obj_and_decref(space, py_traceback) if w_value is None or space.is_w(w_value, space.w_None): operror = None else: @@ -453,9 +457,6 @@ # ec = space.getexecutioncontext() ec.set_sys_exc_info(operror) - Py_DecRef(space, w_type) - Py_DecRef(space, w_value) - Py_DecRef(space, w_traceback) @cpython_api([], rffi.INT_real, error=CANNOT_FAIL) def PyOS_InterruptOccurred(space): diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -7,13 +7,13 @@ from pypy.module.cpyext.api import ( cpython_api, bootstrap_function, PyObject, PyObjectP, ADDR, CANNOT_FAIL, Py_TPFLAGS_HEAPTYPE, PyTypeObjectPtr, is_PyObject, - PyVarObject) + PyVarObject, Py_ssize_t, init_function, cts) from pypy.module.cpyext.state import State from pypy.objspace.std.typeobject import W_TypeObject from pypy.objspace.std.objectobject import W_ObjectObject -from rpython.rlib.objectmodel import specialize +from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.objectmodel import keepalive_until_here -from rpython.rtyper.annlowlevel import llhelper +from rpython.rtyper.annlowlevel import llhelper, cast_instance_to_base_ptr from rpython.rlib import rawrefcount, jit from rpython.rlib.debug import ll_assert, fatalerror @@ -26,9 +26,10 @@ W_BaseObject = W_ObjectObject def get_dealloc(self, space): - from pypy.module.cpyext.typeobject import subtype_dealloc - return subtype_dealloc.api_func.get_llhelper(space) + state = space.fromcache(State) + return state.C._PyPy_subtype_dealloc + # CCC port to C def allocate(self, space, w_type, itemcount=0, immortal=False): # typically called from PyType_GenericAlloc via typedescr.allocate # this returns a PyObject with ob_refcnt == 1. @@ -39,7 +40,7 @@ # Don't increase refcount for non-heaptypes flags = rffi.cast(lltype.Signed, pytype.c_tp_flags) if flags & Py_TPFLAGS_HEAPTYPE: - Py_IncRef(space, w_type) + incref(space, pytype) if pytype: size = pytype.c_tp_basicsize @@ -97,17 +98,21 @@ assert not kw, "Extra arguments to make_typedescr" null_dealloc = lltype.nullptr(lltype.FuncType([PyObject], lltype.Void)) + assert not isinstance(tp_basestruct, lltype.Ptr), "should pass .TO" class CpyTypedescr(BaseCpyTypedescr): basestruct = tp_basestruct if tp_alloc: def allocate(self, space, w_type, itemcount=0, immortal=False): - return tp_alloc(space, w_type, itemcount) + return tp_alloc(self, space, w_type, itemcount) - if tp_dealloc: + if hasattr(tp_dealloc, 'api_func'): def get_dealloc(self, space): return tp_dealloc.api_func.get_llhelper(space) + elif tp_dealloc: + def get_dealloc(self, space): + return tp_dealloc if tp_attach: def attach(self, space, pyobj, w_obj, w_userdata=None): @@ -123,10 +128,10 @@ @bootstrap_function def init_pyobject(space): - from pypy.module.cpyext.object import PyObject_dealloc # typedescr for the 'object' type + state = space.fromcache(State) make_typedescr(space.w_object.layout.typedef, - dealloc=PyObject_dealloc) + dealloc=state.C._PyPy_object_dealloc) # almost all types, which should better inherit from object. make_typedescr(None) @@ -180,7 +185,7 @@ """ Ties together a PyObject and an interpreter object. The PyObject's refcnt is increased by REFCNT_FROM_PYPY. - The reference in 'py_obj' is not stolen! Remember to Py_DecRef() + The reference in 'py_obj' is not stolen! Remember to decref() it is you need to. """ # XXX looks like a PyObject_GC_TRACK @@ -238,8 +243,8 @@ use keepalive_until_here(w_obj) some time later.** In case of doubt, use the safer make_ref(). """ + assert not is_pyobj(w_obj) if w_obj is not None: - assert not is_pyobj(w_obj) py_obj = rawrefcount.from_obj(PyObject, w_obj) if not py_obj: py_obj = create_ref(space, w_obj, w_userdata, immortal=immortal) @@ -277,37 +282,44 @@ hop.exception_cannot_occur() return hop.inputconst(lltype.Bool, hop.s_result.const) - at specialize.ll() -def make_ref(space, obj, w_userdata=None, immortal=False): - """Increment the reference counter of the PyObject and return it. - Can be called with either a PyObject or a W_Root. - """ - if is_pyobj(obj): - pyobj = rffi.cast(PyObject, obj) - at_least = 1 - else: - pyobj = as_pyobj(space, obj, w_userdata, immortal=immortal) - at_least = rawrefcount.REFCNT_FROM_PYPY - if pyobj: - assert pyobj.c_ob_refcnt >= at_least +def get_pyobj_and_incref(space, w_obj, w_userdata=None, immortal=False): + pyobj = as_pyobj(space, w_obj, w_userdata, immortal=immortal) + if pyobj: # != NULL + assert pyobj.c_ob_refcnt >= rawrefcount.REFCNT_FROM_PYPY pyobj.c_ob_refcnt += 1 - keepalive_until_here(obj) + keepalive_until_here(w_obj) return pyobj +def hack_for_result_often_existing_obj(space, w_obj): + # Equivalent to get_pyobj_and_incref() and not to make_ref(): + # it builds a PyObject from a W_Root, but ensures that the result + # gets attached to the original W_Root. This is needed to work around + # some obscure abuses: https://github.com/numpy/numpy/issues/9850 + return get_pyobj_and_incref(space, w_obj) + +def make_ref(space, w_obj, w_userdata=None, immortal=False): + """Turn the W_Root into a corresponding PyObject. You should + decref the returned PyObject later. Note that it is often the + case, but not guaranteed, that make_ref() returns always the + same PyObject for the same W_Root; for example, integers. + """ + assert not is_pyobj(w_obj) + if False and w_obj is not None and space.type(w_obj) is space.w_int: + # XXX: adapt for pypy3 + state = space.fromcache(State) + intval = space.int_w(w_obj) + return state.ccall("PyInt_FromLong", intval) + return get_pyobj_and_incref(space, w_obj, w_userdata, immortal=False) @specialize.ll() -def get_w_obj_and_decref(space, obj): +def get_w_obj_and_decref(space, pyobj): """Decrement the reference counter of the PyObject and return the - corresponding W_Root object (so the reference count is at least - REFCNT_FROM_PYPY and cannot be zero). Can be called with either - a PyObject or a W_Root. + corresponding W_Root object (so the reference count after the decref + is at least REFCNT_FROM_PYPY and cannot be zero). """ - if is_pyobj(obj): - pyobj = rffi.cast(PyObject, obj) - w_obj = from_ref(space, pyobj) - else: - w_obj = obj - pyobj = as_pyobj(space, w_obj) + assert is_pyobj(pyobj) + pyobj = rffi.cast(PyObject, pyobj) + w_obj = from_ref(space, pyobj) if pyobj: pyobj.c_ob_refcnt -= 1 assert pyobj.c_ob_refcnt >= rawrefcount.REFCNT_FROM_PYPY @@ -316,51 +328,37 @@ @specialize.ll() -def incref(space, obj): - make_ref(space, obj) +def incref(space, pyobj): + assert is_pyobj(pyobj) + pyobj = rffi.cast(PyObject, pyobj) + assert pyobj.c_ob_refcnt >= 1 + pyobj.c_ob_refcnt += 1 @specialize.ll() -def decref(space, obj): - if is_pyobj(obj): - obj = rffi.cast(PyObject, obj) - if obj: - assert obj.c_ob_refcnt > 0 - assert obj.c_ob_pypy_link == 0 or obj.c_ob_refcnt > rawrefcount.REFCNT_FROM_PYPY - obj.c_ob_refcnt -= 1 - if obj.c_ob_refcnt == 0: - _Py_Dealloc(space, obj) - #else: - # w_obj = rawrefcount.to_obj(W_Root, ref) - # if w_obj is not None: - # assert obj.c_ob_refcnt >= rawrefcount.REFCNT_FROM_PYPY - else: - get_w_obj_and_decref(space, obj) +def decref(space, pyobj): + from pypy.module.cpyext.api import generic_cpy_call + assert is_pyobj(pyobj) + pyobj = rffi.cast(PyObject, pyobj) + if pyobj: + assert pyobj.c_ob_refcnt > 0 + assert (pyobj.c_ob_pypy_link == 0 or + pyobj.c_ob_refcnt > rawrefcount.REFCNT_FROM_PYPY) + pyobj.c_ob_refcnt -= 1 + if pyobj.c_ob_refcnt == 0: + state = space.fromcache(State) + generic_cpy_call(space, state.C._Py_Dealloc, pyobj) + #else: + # w_obj = rawrefcount.to_obj(W_Root, ref) + # if w_obj is not None: + # assert pyobj.c_ob_refcnt >= rawrefcount.REFCNT_FROM_PYPY - at cpython_api([PyObject], lltype.Void) -def Py_IncRef(space, obj): - incref(space, obj) - - at cpython_api([PyObject], lltype.Void) -def Py_DecRef(space, obj): - decref(space, obj) - - at cpython_api([PyObject], lltype.Void) -def _Py_NewReference(space, obj): - obj.c_ob_refcnt = 1 - # XXX is it always useful to create the W_Root object here? - w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type)) - assert isinstance(w_type, W_TypeObject) - get_typedescr(w_type.layout.typedef).realize(space, obj) - - at cpython_api([PyObject], lltype.Void) -def _Py_Dealloc(space, obj): - from pypy.module.cpyext.api import generic_cpy_call - pto = obj.c_ob_type - #print >>sys.stderr, "Calling dealloc slot", pto.c_tp_dealloc, "of", obj, \ - # "'s type which is", rffi.charp2str(pto.c_tp_name) - rawrefcount.mark_deallocating(w_marker_deallocating, obj) - generic_cpy_call(space, pto.c_tp_dealloc, obj) + at init_function +def write_w_marker_deallocating(space): + if we_are_translated(): + llptr = cast_instance_to_base_ptr(w_marker_deallocating) + state = space.fromcache(State) + state.C.set_marker(rffi.cast(Py_ssize_t, llptr)) @cpython_api([rffi.VOIDP], lltype.Signed, error=CANNOT_FAIL) def _Py_HashPointer(space, ptr): diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py --- a/pypy/module/cpyext/pystate.py +++ b/pypy/module/cpyext/pystate.py @@ -1,6 +1,6 @@ from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, cpython_struct) -from pypy.module.cpyext.pyobject import PyObject, Py_DecRef, make_ref +from pypy.module.cpyext.pyobject import PyObject, decref, make_ref from pypy.interpreter.error import OperationError from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rlib import rthread @@ -75,7 +75,7 @@ def ThreadState_dealloc(ts, space): assert space is not None - Py_DecRef(space, ts.c_dict) + decref(space, ts.c_dict) ThreadStateCapsule = encapsulator(PyThreadState.TO, dealloc=ThreadState_dealloc) @@ -323,7 +323,7 @@ interpreter lock must be held.""" if not space.config.translation.thread: raise NoThreads - Py_DecRef(space, tstate.c_dict) + decref(space, tstate.c_dict) tstate.c_dict = lltype.nullptr(PyObject.TO) space.threadlocals.leave_thread(space) space.getexecutioncontext().cleanup_cpyext_state() diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py --- a/pypy/module/cpyext/pytraceback.py +++ b/pypy/module/cpyext/pytraceback.py @@ -4,7 +4,7 @@ cpython_api, bootstrap_function, cpython_struct, slot_function) from pypy.module.cpyext.pyobject import ( - PyObject, make_ref, from_ref, Py_DecRef, make_typedescr) + PyObject, make_ref, from_ref, decref, make_typedescr) from pypy.module.cpyext.frameobject import PyFrameObject from pypy.interpreter.error import OperationError from pypy.interpreter.pytraceback import PyTraceback @@ -44,7 +44,7 @@ @slot_function([PyObject], lltype.Void) def traceback_dealloc(space, py_obj): py_traceback = rffi.cast(PyTracebackObject, py_obj) - Py_DecRef(space, rffi.cast(PyObject, py_traceback.c_tb_next)) - Py_DecRef(space, rffi.cast(PyObject, py_traceback.c_tb_frame)) + decref(space, rffi.cast(PyObject, py_traceback.c_tb_next)) + decref(space, rffi.cast(PyObject, py_traceback.c_tb_frame)) from pypy.module.cpyext.object import _dealloc _dealloc(space, py_obj) diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py --- a/pypy/module/cpyext/sequence.py +++ b/pypy/module/cpyext/sequence.py @@ -1,11 +1,13 @@ from rpython.rlib import rerased, jit +from rpython.rlib.objectmodel import keepalive_until_here from pypy.interpreter.error import OperationError, oefmt from pypy.objspace.std.listobject import ( ListStrategy, UNROLL_CUTOFF, W_ListObject, ObjectListStrategy) from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, CONST_STRING, Py_ssize_t, PyObject, PyObjectP) from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref +from pypy.module.cpyext.pyobject import as_pyobj, incref from rpython.rtyper.lltypesystem import rffi, lltype from pypy.objspace.std import tupleobject @@ -47,6 +49,8 @@ converted to a sequence, and raises a TypeError, raise a new TypeError with m as the message text. If the conversion otherwise, fails, reraise the original exception""" + if isinstance(w_obj, tupleobject.W_TupleObject): + return w_obj # CCC avoid the double conversion that occurs here if isinstance(w_obj, W_ListObject): # make sure we can return a borrowed obj from PySequence_Fast_GET_ITEM w_obj.convert_to_cpy_strategy(space) @@ -58,6 +62,7 @@ raise OperationError(space.w_TypeError, space.newtext(rffi.charp2str(m))) raise e +# CCC this should be written as a C macro @cpython_api([rffi.VOIDP, Py_ssize_t], PyObject, result_borrowed=True) def PySequence_Fast_GET_ITEM(space, w_obj, index): """Return the ith element of o, assuming that o was returned by @@ -99,6 +104,11 @@ cpy_strategy = space.fromcache(CPyListStrategy) if w_obj.strategy is cpy_strategy: return w_obj.get_raw_items() # asserts it's a cpyext strategy + elif isinstance(w_obj, tupleobject.W_TupleObject): + from pypy.module.cpyext.tupleobject import PyTupleObject + py_obj = as_pyobj(space, w_obj) + py_tuple = rffi.cast(PyTupleObject, py_obj) + return rffi.cast(PyObjectP, py_tuple.c_ob_item) raise oefmt(space.w_TypeError, "PySequence_Fast_ITEMS called but object is not the result of " "PySequence_Fast") @@ -123,7 +133,7 @@ space.delslice(w_obj, space.newint(start), space.newint(end)) return 0 - at cpython_api([rffi.VOIDP, Py_ssize_t], PyObject) + at cpython_api([rffi.VOIDP, Py_ssize_t], PyObject, result_is_ll=True) def PySequence_ITEM(space, w_obj, i): """Return the ith element of o or NULL on failure. Macro form of PySequence_GetItem() but without checking that @@ -132,12 +142,32 @@ This function used an int type for i. This might require changes in your code for properly supporting 64-bit systems.""" - return space.getitem(w_obj, space.newint(i)) + # XXX we should call Py*_GET_ITEM() instead of Py*_GetItem() + # from here, but we cannot because we are also called from + # PySequence_GetItem() + if isinstance(w_obj, tupleobject.W_TupleObject): + from pypy.module.cpyext.tupleobject import PyTuple_GetItem + py_obj = as_pyobj(space, w_obj) + py_res = PyTuple_GetItem(space, py_obj, i) + incref(space, py_res) + keepalive_until_here(w_obj) + return py_res + if isinstance(w_obj, W_ListObject): + from pypy.module.cpyext.listobject import PyList_GetItem + py_obj = as_pyobj(space, w_obj) + py_res = PyList_GetItem(space, py_obj, i) + incref(space, py_res) + keepalive_until_here(w_obj) + return py_res + return make_ref(space, space.getitem(w_obj, space.newint(i))) - at cpython_api([PyObject, Py_ssize_t], PyObject) + at cpython_api([PyObject, Py_ssize_t], PyObject, result_is_ll=True) def PySequence_GetItem(space, w_obj, i): """Return the ith element of o, or NULL on failure. This is the equivalent of the Python expression o[i].""" + if i < 0: + l = PySequence_Length(space, w_obj) + i += l return PySequence_ITEM(space, w_obj, i) @cpython_api([PyObject], PyObject) diff --git a/pypy/module/cpyext/setobject.py b/pypy/module/cpyext/setobject.py --- a/pypy/module/cpyext/setobject.py +++ b/pypy/module/cpyext/setobject.py @@ -2,7 +2,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL, build_type_checkers) -from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, Py_DecRef, +from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, make_ref, from_ref) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.objspace.std.setobject import W_SetObject, W_FrozensetObject, newset diff --git a/pypy/module/cpyext/sliceobject.py b/pypy/module/cpyext/sliceobject.py --- a/pypy/module/cpyext/sliceobject.py +++ b/pypy/module/cpyext/sliceobject.py @@ -3,7 +3,7 @@ cpython_api, cpython_struct, bootstrap_function, build_type_checkers, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP, PyObjectFields, slot_function) from pypy.module.cpyext.pyobject import ( - Py_DecRef, PyObject, make_ref, make_typedescr) + decref, PyObject, make_ref, make_typedescr) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.interpreter.error import OperationError from pypy.objspace.std.sliceobject import W_SliceObject @@ -41,9 +41,9 @@ """Frees allocated PySliceObject resources. """ py_slice = rffi.cast(PySliceObject, py_obj) - Py_DecRef(space, py_slice.c_start) - Py_DecRef(space, py_slice.c_stop) - Py_DecRef(space, py_slice.c_step) + decref(space, py_slice.c_start) + decref(space, py_slice.c_stop) + decref(space, py_slice.c_step) from pypy.module.cpyext.object import _dealloc _dealloc(space, py_obj) diff --git a/pypy/module/cpyext/src/intobject.c b/pypy/module/cpyext/src/intobject.c new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/src/intobject.c @@ -0,0 +1,108 @@ + +/* Integer object implementation -- copied&adapted from CPython */ + +#include "Python.h" + +/* Integers are quite normal objects, to make object handling uniform. + (Using odd pointers to represent integers would save much space + but require extra checks for this special case throughout the code.) + Since a typical Python program spends much of its time allocating + and deallocating integers, these operations should be very fast. + Therefore we use a dedicated allocation scheme with a much lower + overhead (in space and time) than straight malloc(): a simple + dedicated free list, filled when necessary with memory from malloc(). + + block_list is a singly-linked list of all PyIntBlocks ever allocated, + linked via their next members. PyIntBlocks are never returned to the + system before shutdown (PyInt_Fini). + + free_list is a singly-linked list of available PyIntObjects, linked + via abuse of their ob_type members. +*/ + +#define BLOCK_SIZE 1000 /* 1K less typical malloc overhead */ +#define BHEAD_SIZE 8 /* Enough for a 64-bit pointer */ +#define N_INTOBJECTS ((BLOCK_SIZE - BHEAD_SIZE) / sizeof(PyIntObject)) + +struct _intblock { + struct _intblock *next; + PyIntObject objects[N_INTOBJECTS]; +}; + +typedef struct _intblock PyIntBlock; + +static PyIntBlock *block_list = NULL; +static PyIntObject *free_list = NULL; + +static PyIntObject * +fill_free_list(void) +{ + PyIntObject *p, *q; + /* Python's object allocator isn't appropriate for large blocks. */ + p = (PyIntObject *) PyMem_MALLOC(sizeof(PyIntBlock)); + if (p == NULL) + return (PyIntObject *) PyErr_NoMemory(); + ((PyIntBlock *)p)->next = block_list; + block_list = (PyIntBlock *)p; + /* Link the int objects together, from rear to front, then return + the address of the last int object in the block. */ + p = &((PyIntBlock *)p)->objects[0]; + q = p + N_INTOBJECTS; + while (--q > p) + Py_TYPE(q) = (struct _typeobject *)(q-1); + Py_TYPE(q) = NULL; + return p + N_INTOBJECTS - 1; +} + +#ifndef NSMALLPOSINTS +#define NSMALLPOSINTS 257 +#endif +#ifndef NSMALLNEGINTS +#define NSMALLNEGINTS 5 +#endif +#if NSMALLNEGINTS + NSMALLPOSINTS > 0 +/* References to small integers are saved in this array so that they + can be shared. + The integers that are saved are those in the range + -NSMALLNEGINTS (inclusive) to NSMALLPOSINTS (not inclusive). +*/ +static PyIntObject *small_ints[NSMALLNEGINTS + NSMALLPOSINTS]; +#endif + +PyObject * +PyInt_FromLong(long ival) +{ + register PyIntObject *v; + /* +#if NSMALLNEGINTS + NSMALLPOSINTS > 0 + if (-NSMALLNEGINTS <= ival && ival < NSMALLPOSINTS) { + v = small_ints[ival + NSMALLNEGINTS]; + Py_INCREF(v); + return (PyObject *) v; + } +#endif + */ + if (free_list == NULL) { + if ((free_list = fill_free_list()) == NULL) + return NULL; + } + /* Inline PyObject_New */ + v = free_list; + free_list = (PyIntObject *)Py_TYPE(v); + (void)PyObject_INIT(v, &PyInt_Type); + v->ob_ival = ival; + return (PyObject *) v; +} + +/* this is CPython's int_dealloc */ +void +_PyPy_int_dealloc(PyObject *obj) +{ + PyIntObject *v = (PyIntObject *)obj; + if (PyInt_CheckExact(v)) { + Py_TYPE(v) = (struct _typeobject *)free_list; From pypy.commits at gmail.com Sat Jan 6 10:46:11 2018 From: pypy.commits at gmail.com (amauryfa) Date: Sat, 06 Jan 2018 07:46:11 -0800 (PST) Subject: [pypy-commit] pypy default: Add Unicode database 9.0.0 Message-ID: <5a50ef43.c6a51c0a.abe59.385c@mx.google.com> Author: Amaury Forgeot d'Arc Branch: Changeset: r93629:912f3925af42 Date: 2018-01-04 17:44 +0100 http://bitbucket.org/pypy/pypy/changeset/912f3925af42/ Log: Add Unicode database 9.0.0 diff too long, truncating to 2000 out of 75520 lines diff --git a/rpython/rlib/unicodedata/CaseFolding-9.0.0.txt b/rpython/rlib/unicodedata/CaseFolding-9.0.0.txt new file mode 100644 --- /dev/null +++ b/rpython/rlib/unicodedata/CaseFolding-9.0.0.txt @@ -0,0 +1,1495 @@ +# CaseFolding-9.0.0.txt +# Date: 2016-03-02, 18:54:54 GMT +# © 2016 Unicode®, Inc. +# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. +# For terms of use, see http://www.unicode.org/terms_of_use.html +# +# Unicode Character Database +# For documentation, see http://www.unicode.org/reports/tr44/ +# +# Case Folding Properties +# +# This file is a supplement to the UnicodeData file. +# It provides a case folding mapping generated from the Unicode Character Database. +# If all characters are mapped according to the full mapping below, then +# case differences (according to UnicodeData.txt and SpecialCasing.txt) +# are eliminated. +# +# The data supports both implementations that require simple case foldings +# (where string lengths don't change), and implementations that allow full case folding +# (where string lengths may grow). Note that where they can be supported, the +# full case foldings are superior: for example, they allow "MASSE" and "Maße" to match. +# +# All code points not listed in this file map to themselves. +# +# NOTE: case folding does not preserve normalization formats! +# +# For information on case folding, including how to have case folding +# preserve normalization formats, see Section 3.13 Default Case Algorithms in +# The Unicode Standard. +# +# ================================================================================ +# Format +# ================================================================================ +# The entries in this file are in the following machine-readable format: +# +# ; ; ; # +# +# The status field is: +# C: common case folding, common mappings shared by both simple and full mappings. +# F: full case folding, mappings that cause strings to grow in length. Multiple characters are separated by spaces. +# S: simple case folding, mappings to single characters where different from F. +# T: special case for uppercase I and dotted uppercase I +# - For non-Turkic languages, this mapping is normally not used. +# - For Turkic languages (tr, az), this mapping can be used instead of the normal mapping for these characters. +# Note that the Turkic mappings do not maintain canonical equivalence without additional processing. +# See the discussions of case mapping in the Unicode Standard for more information. +# +# Usage: +# A. To do a simple case folding, use the mappings with status C + S. +# B. To do a full case folding, use the mappings with status C + F. +# +# The mappings with status T can be used or omitted depending on the desired case-folding +# behavior. (The default option is to exclude them.) +# +# ================================================================= + +# Property: Case_Folding + +# All code points not explicitly listed for Case_Folding +# have the value C for the status field, and the code point itself for the mapping field. + +# ================================================================= +0041; C; 0061; # LATIN CAPITAL LETTER A +0042; C; 0062; # LATIN CAPITAL LETTER B +0043; C; 0063; # LATIN CAPITAL LETTER C +0044; C; 0064; # LATIN CAPITAL LETTER D +0045; C; 0065; # LATIN CAPITAL LETTER E +0046; C; 0066; # LATIN CAPITAL LETTER F +0047; C; 0067; # LATIN CAPITAL LETTER G +0048; C; 0068; # LATIN CAPITAL LETTER H +0049; C; 0069; # LATIN CAPITAL LETTER I +0049; T; 0131; # LATIN CAPITAL LETTER I +004A; C; 006A; # LATIN CAPITAL LETTER J +004B; C; 006B; # LATIN CAPITAL LETTER K +004C; C; 006C; # LATIN CAPITAL LETTER L +004D; C; 006D; # LATIN CAPITAL LETTER M +004E; C; 006E; # LATIN CAPITAL LETTER N +004F; C; 006F; # LATIN CAPITAL LETTER O +0050; C; 0070; # LATIN CAPITAL LETTER P +0051; C; 0071; # LATIN CAPITAL LETTER Q +0052; C; 0072; # LATIN CAPITAL LETTER R +0053; C; 0073; # LATIN CAPITAL LETTER S +0054; C; 0074; # LATIN CAPITAL LETTER T +0055; C; 0075; # LATIN CAPITAL LETTER U +0056; C; 0076; # LATIN CAPITAL LETTER V +0057; C; 0077; # LATIN CAPITAL LETTER W +0058; C; 0078; # LATIN CAPITAL LETTER X +0059; C; 0079; # LATIN CAPITAL LETTER Y +005A; C; 007A; # LATIN CAPITAL LETTER Z +00B5; C; 03BC; # MICRO SIGN +00C0; C; 00E0; # LATIN CAPITAL LETTER A WITH GRAVE +00C1; C; 00E1; # LATIN CAPITAL LETTER A WITH ACUTE +00C2; C; 00E2; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX +00C3; C; 00E3; # LATIN CAPITAL LETTER A WITH TILDE +00C4; C; 00E4; # LATIN CAPITAL LETTER A WITH DIAERESIS +00C5; C; 00E5; # LATIN CAPITAL LETTER A WITH RING ABOVE +00C6; C; 00E6; # LATIN CAPITAL LETTER AE +00C7; C; 00E7; # LATIN CAPITAL LETTER C WITH CEDILLA +00C8; C; 00E8; # LATIN CAPITAL LETTER E WITH GRAVE +00C9; C; 00E9; # LATIN CAPITAL LETTER E WITH ACUTE +00CA; C; 00EA; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX +00CB; C; 00EB; # LATIN CAPITAL LETTER E WITH DIAERESIS +00CC; C; 00EC; # LATIN CAPITAL LETTER I WITH GRAVE +00CD; C; 00ED; # LATIN CAPITAL LETTER I WITH ACUTE +00CE; C; 00EE; # LATIN CAPITAL LETTER I WITH CIRCUMFLEX +00CF; C; 00EF; # LATIN CAPITAL LETTER I WITH DIAERESIS +00D0; C; 00F0; # LATIN CAPITAL LETTER ETH +00D1; C; 00F1; # LATIN CAPITAL LETTER N WITH TILDE +00D2; C; 00F2; # LATIN CAPITAL LETTER O WITH GRAVE +00D3; C; 00F3; # LATIN CAPITAL LETTER O WITH ACUTE +00D4; C; 00F4; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX +00D5; C; 00F5; # LATIN CAPITAL LETTER O WITH TILDE +00D6; C; 00F6; # LATIN CAPITAL LETTER O WITH DIAERESIS +00D8; C; 00F8; # LATIN CAPITAL LETTER O WITH STROKE +00D9; C; 00F9; # LATIN CAPITAL LETTER U WITH GRAVE +00DA; C; 00FA; # LATIN CAPITAL LETTER U WITH ACUTE +00DB; C; 00FB; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX +00DC; C; 00FC; # LATIN CAPITAL LETTER U WITH DIAERESIS +00DD; C; 00FD; # LATIN CAPITAL LETTER Y WITH ACUTE +00DE; C; 00FE; # LATIN CAPITAL LETTER THORN +00DF; F; 0073 0073; # LATIN SMALL LETTER SHARP S +0100; C; 0101; # LATIN CAPITAL LETTER A WITH MACRON +0102; C; 0103; # LATIN CAPITAL LETTER A WITH BREVE +0104; C; 0105; # LATIN CAPITAL LETTER A WITH OGONEK +0106; C; 0107; # LATIN CAPITAL LETTER C WITH ACUTE +0108; C; 0109; # LATIN CAPITAL LETTER C WITH CIRCUMFLEX +010A; C; 010B; # LATIN CAPITAL LETTER C WITH DOT ABOVE +010C; C; 010D; # LATIN CAPITAL LETTER C WITH CARON +010E; C; 010F; # LATIN CAPITAL LETTER D WITH CARON +0110; C; 0111; # LATIN CAPITAL LETTER D WITH STROKE +0112; C; 0113; # LATIN CAPITAL LETTER E WITH MACRON +0114; C; 0115; # LATIN CAPITAL LETTER E WITH BREVE +0116; C; 0117; # LATIN CAPITAL LETTER E WITH DOT ABOVE +0118; C; 0119; # LATIN CAPITAL LETTER E WITH OGONEK +011A; C; 011B; # LATIN CAPITAL LETTER E WITH CARON +011C; C; 011D; # LATIN CAPITAL LETTER G WITH CIRCUMFLEX +011E; C; 011F; # LATIN CAPITAL LETTER G WITH BREVE +0120; C; 0121; # LATIN CAPITAL LETTER G WITH DOT ABOVE +0122; C; 0123; # LATIN CAPITAL LETTER G WITH CEDILLA +0124; C; 0125; # LATIN CAPITAL LETTER H WITH CIRCUMFLEX +0126; C; 0127; # LATIN CAPITAL LETTER H WITH STROKE +0128; C; 0129; # LATIN CAPITAL LETTER I WITH TILDE +012A; C; 012B; # LATIN CAPITAL LETTER I WITH MACRON +012C; C; 012D; # LATIN CAPITAL LETTER I WITH BREVE +012E; C; 012F; # LATIN CAPITAL LETTER I WITH OGONEK +0130; F; 0069 0307; # LATIN CAPITAL LETTER I WITH DOT ABOVE +0130; T; 0069; # LATIN CAPITAL LETTER I WITH DOT ABOVE +0132; C; 0133; # LATIN CAPITAL LIGATURE IJ +0134; C; 0135; # LATIN CAPITAL LETTER J WITH CIRCUMFLEX +0136; C; 0137; # LATIN CAPITAL LETTER K WITH CEDILLA +0139; C; 013A; # LATIN CAPITAL LETTER L WITH ACUTE +013B; C; 013C; # LATIN CAPITAL LETTER L WITH CEDILLA +013D; C; 013E; # LATIN CAPITAL LETTER L WITH CARON +013F; C; 0140; # LATIN CAPITAL LETTER L WITH MIDDLE DOT +0141; C; 0142; # LATIN CAPITAL LETTER L WITH STROKE +0143; C; 0144; # LATIN CAPITAL LETTER N WITH ACUTE +0145; C; 0146; # LATIN CAPITAL LETTER N WITH CEDILLA +0147; C; 0148; # LATIN CAPITAL LETTER N WITH CARON +0149; F; 02BC 006E; # LATIN SMALL LETTER N PRECEDED BY APOSTROPHE +014A; C; 014B; # LATIN CAPITAL LETTER ENG +014C; C; 014D; # LATIN CAPITAL LETTER O WITH MACRON +014E; C; 014F; # LATIN CAPITAL LETTER O WITH BREVE +0150; C; 0151; # LATIN CAPITAL LETTER O WITH DOUBLE ACUTE +0152; C; 0153; # LATIN CAPITAL LIGATURE OE +0154; C; 0155; # LATIN CAPITAL LETTER R WITH ACUTE +0156; C; 0157; # LATIN CAPITAL LETTER R WITH CEDILLA +0158; C; 0159; # LATIN CAPITAL LETTER R WITH CARON +015A; C; 015B; # LATIN CAPITAL LETTER S WITH ACUTE +015C; C; 015D; # LATIN CAPITAL LETTER S WITH CIRCUMFLEX +015E; C; 015F; # LATIN CAPITAL LETTER S WITH CEDILLA +0160; C; 0161; # LATIN CAPITAL LETTER S WITH CARON +0162; C; 0163; # LATIN CAPITAL LETTER T WITH CEDILLA +0164; C; 0165; # LATIN CAPITAL LETTER T WITH CARON +0166; C; 0167; # LATIN CAPITAL LETTER T WITH STROKE +0168; C; 0169; # LATIN CAPITAL LETTER U WITH TILDE +016A; C; 016B; # LATIN CAPITAL LETTER U WITH MACRON +016C; C; 016D; # LATIN CAPITAL LETTER U WITH BREVE +016E; C; 016F; # LATIN CAPITAL LETTER U WITH RING ABOVE +0170; C; 0171; # LATIN CAPITAL LETTER U WITH DOUBLE ACUTE +0172; C; 0173; # LATIN CAPITAL LETTER U WITH OGONEK +0174; C; 0175; # LATIN CAPITAL LETTER W WITH CIRCUMFLEX +0176; C; 0177; # LATIN CAPITAL LETTER Y WITH CIRCUMFLEX +0178; C; 00FF; # LATIN CAPITAL LETTER Y WITH DIAERESIS +0179; C; 017A; # LATIN CAPITAL LETTER Z WITH ACUTE +017B; C; 017C; # LATIN CAPITAL LETTER Z WITH DOT ABOVE +017D; C; 017E; # LATIN CAPITAL LETTER Z WITH CARON +017F; C; 0073; # LATIN SMALL LETTER LONG S +0181; C; 0253; # LATIN CAPITAL LETTER B WITH HOOK +0182; C; 0183; # LATIN CAPITAL LETTER B WITH TOPBAR +0184; C; 0185; # LATIN CAPITAL LETTER TONE SIX +0186; C; 0254; # LATIN CAPITAL LETTER OPEN O +0187; C; 0188; # LATIN CAPITAL LETTER C WITH HOOK +0189; C; 0256; # LATIN CAPITAL LETTER AFRICAN D +018A; C; 0257; # LATIN CAPITAL LETTER D WITH HOOK +018B; C; 018C; # LATIN CAPITAL LETTER D WITH TOPBAR +018E; C; 01DD; # LATIN CAPITAL LETTER REVERSED E +018F; C; 0259; # LATIN CAPITAL LETTER SCHWA +0190; C; 025B; # LATIN CAPITAL LETTER OPEN E +0191; C; 0192; # LATIN CAPITAL LETTER F WITH HOOK +0193; C; 0260; # LATIN CAPITAL LETTER G WITH HOOK +0194; C; 0263; # LATIN CAPITAL LETTER GAMMA +0196; C; 0269; # LATIN CAPITAL LETTER IOTA +0197; C; 0268; # LATIN CAPITAL LETTER I WITH STROKE +0198; C; 0199; # LATIN CAPITAL LETTER K WITH HOOK +019C; C; 026F; # LATIN CAPITAL LETTER TURNED M +019D; C; 0272; # LATIN CAPITAL LETTER N WITH LEFT HOOK +019F; C; 0275; # LATIN CAPITAL LETTER O WITH MIDDLE TILDE +01A0; C; 01A1; # LATIN CAPITAL LETTER O WITH HORN +01A2; C; 01A3; # LATIN CAPITAL LETTER OI +01A4; C; 01A5; # LATIN CAPITAL LETTER P WITH HOOK +01A6; C; 0280; # LATIN LETTER YR +01A7; C; 01A8; # LATIN CAPITAL LETTER TONE TWO +01A9; C; 0283; # LATIN CAPITAL LETTER ESH +01AC; C; 01AD; # LATIN CAPITAL LETTER T WITH HOOK +01AE; C; 0288; # LATIN CAPITAL LETTER T WITH RETROFLEX HOOK +01AF; C; 01B0; # LATIN CAPITAL LETTER U WITH HORN +01B1; C; 028A; # LATIN CAPITAL LETTER UPSILON +01B2; C; 028B; # LATIN CAPITAL LETTER V WITH HOOK +01B3; C; 01B4; # LATIN CAPITAL LETTER Y WITH HOOK +01B5; C; 01B6; # LATIN CAPITAL LETTER Z WITH STROKE +01B7; C; 0292; # LATIN CAPITAL LETTER EZH +01B8; C; 01B9; # LATIN CAPITAL LETTER EZH REVERSED +01BC; C; 01BD; # LATIN CAPITAL LETTER TONE FIVE +01C4; C; 01C6; # LATIN CAPITAL LETTER DZ WITH CARON +01C5; C; 01C6; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON +01C7; C; 01C9; # LATIN CAPITAL LETTER LJ +01C8; C; 01C9; # LATIN CAPITAL LETTER L WITH SMALL LETTER J +01CA; C; 01CC; # LATIN CAPITAL LETTER NJ +01CB; C; 01CC; # LATIN CAPITAL LETTER N WITH SMALL LETTER J +01CD; C; 01CE; # LATIN CAPITAL LETTER A WITH CARON +01CF; C; 01D0; # LATIN CAPITAL LETTER I WITH CARON +01D1; C; 01D2; # LATIN CAPITAL LETTER O WITH CARON +01D3; C; 01D4; # LATIN CAPITAL LETTER U WITH CARON +01D5; C; 01D6; # LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON +01D7; C; 01D8; # LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE +01D9; C; 01DA; # LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON +01DB; C; 01DC; # LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE +01DE; C; 01DF; # LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON +01E0; C; 01E1; # LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON +01E2; C; 01E3; # LATIN CAPITAL LETTER AE WITH MACRON +01E4; C; 01E5; # LATIN CAPITAL LETTER G WITH STROKE +01E6; C; 01E7; # LATIN CAPITAL LETTER G WITH CARON +01E8; C; 01E9; # LATIN CAPITAL LETTER K WITH CARON +01EA; C; 01EB; # LATIN CAPITAL LETTER O WITH OGONEK +01EC; C; 01ED; # LATIN CAPITAL LETTER O WITH OGONEK AND MACRON +01EE; C; 01EF; # LATIN CAPITAL LETTER EZH WITH CARON +01F0; F; 006A 030C; # LATIN SMALL LETTER J WITH CARON +01F1; C; 01F3; # LATIN CAPITAL LETTER DZ +01F2; C; 01F3; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z +01F4; C; 01F5; # LATIN CAPITAL LETTER G WITH ACUTE +01F6; C; 0195; # LATIN CAPITAL LETTER HWAIR +01F7; C; 01BF; # LATIN CAPITAL LETTER WYNN +01F8; C; 01F9; # LATIN CAPITAL LETTER N WITH GRAVE +01FA; C; 01FB; # LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE +01FC; C; 01FD; # LATIN CAPITAL LETTER AE WITH ACUTE +01FE; C; 01FF; # LATIN CAPITAL LETTER O WITH STROKE AND ACUTE +0200; C; 0201; # LATIN CAPITAL LETTER A WITH DOUBLE GRAVE +0202; C; 0203; # LATIN CAPITAL LETTER A WITH INVERTED BREVE +0204; C; 0205; # LATIN CAPITAL LETTER E WITH DOUBLE GRAVE +0206; C; 0207; # LATIN CAPITAL LETTER E WITH INVERTED BREVE +0208; C; 0209; # LATIN CAPITAL LETTER I WITH DOUBLE GRAVE +020A; C; 020B; # LATIN CAPITAL LETTER I WITH INVERTED BREVE +020C; C; 020D; # LATIN CAPITAL LETTER O WITH DOUBLE GRAVE +020E; C; 020F; # LATIN CAPITAL LETTER O WITH INVERTED BREVE +0210; C; 0211; # LATIN CAPITAL LETTER R WITH DOUBLE GRAVE +0212; C; 0213; # LATIN CAPITAL LETTER R WITH INVERTED BREVE +0214; C; 0215; # LATIN CAPITAL LETTER U WITH DOUBLE GRAVE +0216; C; 0217; # LATIN CAPITAL LETTER U WITH INVERTED BREVE +0218; C; 0219; # LATIN CAPITAL LETTER S WITH COMMA BELOW +021A; C; 021B; # LATIN CAPITAL LETTER T WITH COMMA BELOW +021C; C; 021D; # LATIN CAPITAL LETTER YOGH +021E; C; 021F; # LATIN CAPITAL LETTER H WITH CARON +0220; C; 019E; # LATIN CAPITAL LETTER N WITH LONG RIGHT LEG +0222; C; 0223; # LATIN CAPITAL LETTER OU +0224; C; 0225; # LATIN CAPITAL LETTER Z WITH HOOK +0226; C; 0227; # LATIN CAPITAL LETTER A WITH DOT ABOVE +0228; C; 0229; # LATIN CAPITAL LETTER E WITH CEDILLA +022A; C; 022B; # LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON +022C; C; 022D; # LATIN CAPITAL LETTER O WITH TILDE AND MACRON +022E; C; 022F; # LATIN CAPITAL LETTER O WITH DOT ABOVE +0230; C; 0231; # LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON +0232; C; 0233; # LATIN CAPITAL LETTER Y WITH MACRON +023A; C; 2C65; # LATIN CAPITAL LETTER A WITH STROKE +023B; C; 023C; # LATIN CAPITAL LETTER C WITH STROKE +023D; C; 019A; # LATIN CAPITAL LETTER L WITH BAR +023E; C; 2C66; # LATIN CAPITAL LETTER T WITH DIAGONAL STROKE +0241; C; 0242; # LATIN CAPITAL LETTER GLOTTAL STOP +0243; C; 0180; # LATIN CAPITAL LETTER B WITH STROKE +0244; C; 0289; # LATIN CAPITAL LETTER U BAR +0245; C; 028C; # LATIN CAPITAL LETTER TURNED V +0246; C; 0247; # LATIN CAPITAL LETTER E WITH STROKE +0248; C; 0249; # LATIN CAPITAL LETTER J WITH STROKE +024A; C; 024B; # LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL +024C; C; 024D; # LATIN CAPITAL LETTER R WITH STROKE +024E; C; 024F; # LATIN CAPITAL LETTER Y WITH STROKE +0345; C; 03B9; # COMBINING GREEK YPOGEGRAMMENI +0370; C; 0371; # GREEK CAPITAL LETTER HETA +0372; C; 0373; # GREEK CAPITAL LETTER ARCHAIC SAMPI +0376; C; 0377; # GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA +037F; C; 03F3; # GREEK CAPITAL LETTER YOT +0386; C; 03AC; # GREEK CAPITAL LETTER ALPHA WITH TONOS +0388; C; 03AD; # GREEK CAPITAL LETTER EPSILON WITH TONOS +0389; C; 03AE; # GREEK CAPITAL LETTER ETA WITH TONOS +038A; C; 03AF; # GREEK CAPITAL LETTER IOTA WITH TONOS +038C; C; 03CC; # GREEK CAPITAL LETTER OMICRON WITH TONOS +038E; C; 03CD; # GREEK CAPITAL LETTER UPSILON WITH TONOS +038F; C; 03CE; # GREEK CAPITAL LETTER OMEGA WITH TONOS +0390; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS +0391; C; 03B1; # GREEK CAPITAL LETTER ALPHA +0392; C; 03B2; # GREEK CAPITAL LETTER BETA +0393; C; 03B3; # GREEK CAPITAL LETTER GAMMA +0394; C; 03B4; # GREEK CAPITAL LETTER DELTA +0395; C; 03B5; # GREEK CAPITAL LETTER EPSILON +0396; C; 03B6; # GREEK CAPITAL LETTER ZETA +0397; C; 03B7; # GREEK CAPITAL LETTER ETA +0398; C; 03B8; # GREEK CAPITAL LETTER THETA +0399; C; 03B9; # GREEK CAPITAL LETTER IOTA +039A; C; 03BA; # GREEK CAPITAL LETTER KAPPA +039B; C; 03BB; # GREEK CAPITAL LETTER LAMDA +039C; C; 03BC; # GREEK CAPITAL LETTER MU +039D; C; 03BD; # GREEK CAPITAL LETTER NU +039E; C; 03BE; # GREEK CAPITAL LETTER XI +039F; C; 03BF; # GREEK CAPITAL LETTER OMICRON +03A0; C; 03C0; # GREEK CAPITAL LETTER PI +03A1; C; 03C1; # GREEK CAPITAL LETTER RHO +03A3; C; 03C3; # GREEK CAPITAL LETTER SIGMA +03A4; C; 03C4; # GREEK CAPITAL LETTER TAU +03A5; C; 03C5; # GREEK CAPITAL LETTER UPSILON +03A6; C; 03C6; # GREEK CAPITAL LETTER PHI +03A7; C; 03C7; # GREEK CAPITAL LETTER CHI +03A8; C; 03C8; # GREEK CAPITAL LETTER PSI +03A9; C; 03C9; # GREEK CAPITAL LETTER OMEGA +03AA; C; 03CA; # GREEK CAPITAL LETTER IOTA WITH DIALYTIKA +03AB; C; 03CB; # GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA +03B0; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS +03C2; C; 03C3; # GREEK SMALL LETTER FINAL SIGMA +03CF; C; 03D7; # GREEK CAPITAL KAI SYMBOL +03D0; C; 03B2; # GREEK BETA SYMBOL +03D1; C; 03B8; # GREEK THETA SYMBOL +03D5; C; 03C6; # GREEK PHI SYMBOL +03D6; C; 03C0; # GREEK PI SYMBOL +03D8; C; 03D9; # GREEK LETTER ARCHAIC KOPPA +03DA; C; 03DB; # GREEK LETTER STIGMA +03DC; C; 03DD; # GREEK LETTER DIGAMMA +03DE; C; 03DF; # GREEK LETTER KOPPA +03E0; C; 03E1; # GREEK LETTER SAMPI +03E2; C; 03E3; # COPTIC CAPITAL LETTER SHEI +03E4; C; 03E5; # COPTIC CAPITAL LETTER FEI +03E6; C; 03E7; # COPTIC CAPITAL LETTER KHEI +03E8; C; 03E9; # COPTIC CAPITAL LETTER HORI +03EA; C; 03EB; # COPTIC CAPITAL LETTER GANGIA +03EC; C; 03ED; # COPTIC CAPITAL LETTER SHIMA +03EE; C; 03EF; # COPTIC CAPITAL LETTER DEI +03F0; C; 03BA; # GREEK KAPPA SYMBOL +03F1; C; 03C1; # GREEK RHO SYMBOL +03F4; C; 03B8; # GREEK CAPITAL THETA SYMBOL +03F5; C; 03B5; # GREEK LUNATE EPSILON SYMBOL +03F7; C; 03F8; # GREEK CAPITAL LETTER SHO +03F9; C; 03F2; # GREEK CAPITAL LUNATE SIGMA SYMBOL +03FA; C; 03FB; # GREEK CAPITAL LETTER SAN +03FD; C; 037B; # GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL +03FE; C; 037C; # GREEK CAPITAL DOTTED LUNATE SIGMA SYMBOL +03FF; C; 037D; # GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL +0400; C; 0450; # CYRILLIC CAPITAL LETTER IE WITH GRAVE +0401; C; 0451; # CYRILLIC CAPITAL LETTER IO +0402; C; 0452; # CYRILLIC CAPITAL LETTER DJE +0403; C; 0453; # CYRILLIC CAPITAL LETTER GJE +0404; C; 0454; # CYRILLIC CAPITAL LETTER UKRAINIAN IE +0405; C; 0455; # CYRILLIC CAPITAL LETTER DZE +0406; C; 0456; # CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I +0407; C; 0457; # CYRILLIC CAPITAL LETTER YI +0408; C; 0458; # CYRILLIC CAPITAL LETTER JE +0409; C; 0459; # CYRILLIC CAPITAL LETTER LJE +040A; C; 045A; # CYRILLIC CAPITAL LETTER NJE +040B; C; 045B; # CYRILLIC CAPITAL LETTER TSHE +040C; C; 045C; # CYRILLIC CAPITAL LETTER KJE +040D; C; 045D; # CYRILLIC CAPITAL LETTER I WITH GRAVE +040E; C; 045E; # CYRILLIC CAPITAL LETTER SHORT U +040F; C; 045F; # CYRILLIC CAPITAL LETTER DZHE +0410; C; 0430; # CYRILLIC CAPITAL LETTER A +0411; C; 0431; # CYRILLIC CAPITAL LETTER BE +0412; C; 0432; # CYRILLIC CAPITAL LETTER VE +0413; C; 0433; # CYRILLIC CAPITAL LETTER GHE +0414; C; 0434; # CYRILLIC CAPITAL LETTER DE +0415; C; 0435; # CYRILLIC CAPITAL LETTER IE +0416; C; 0436; # CYRILLIC CAPITAL LETTER ZHE +0417; C; 0437; # CYRILLIC CAPITAL LETTER ZE +0418; C; 0438; # CYRILLIC CAPITAL LETTER I +0419; C; 0439; # CYRILLIC CAPITAL LETTER SHORT I +041A; C; 043A; # CYRILLIC CAPITAL LETTER KA +041B; C; 043B; # CYRILLIC CAPITAL LETTER EL +041C; C; 043C; # CYRILLIC CAPITAL LETTER EM +041D; C; 043D; # CYRILLIC CAPITAL LETTER EN +041E; C; 043E; # CYRILLIC CAPITAL LETTER O +041F; C; 043F; # CYRILLIC CAPITAL LETTER PE +0420; C; 0440; # CYRILLIC CAPITAL LETTER ER +0421; C; 0441; # CYRILLIC CAPITAL LETTER ES +0422; C; 0442; # CYRILLIC CAPITAL LETTER TE +0423; C; 0443; # CYRILLIC CAPITAL LETTER U +0424; C; 0444; # CYRILLIC CAPITAL LETTER EF +0425; C; 0445; # CYRILLIC CAPITAL LETTER HA +0426; C; 0446; # CYRILLIC CAPITAL LETTER TSE +0427; C; 0447; # CYRILLIC CAPITAL LETTER CHE +0428; C; 0448; # CYRILLIC CAPITAL LETTER SHA +0429; C; 0449; # CYRILLIC CAPITAL LETTER SHCHA +042A; C; 044A; # CYRILLIC CAPITAL LETTER HARD SIGN +042B; C; 044B; # CYRILLIC CAPITAL LETTER YERU +042C; C; 044C; # CYRILLIC CAPITAL LETTER SOFT SIGN +042D; C; 044D; # CYRILLIC CAPITAL LETTER E +042E; C; 044E; # CYRILLIC CAPITAL LETTER YU +042F; C; 044F; # CYRILLIC CAPITAL LETTER YA +0460; C; 0461; # CYRILLIC CAPITAL LETTER OMEGA +0462; C; 0463; # CYRILLIC CAPITAL LETTER YAT +0464; C; 0465; # CYRILLIC CAPITAL LETTER IOTIFIED E +0466; C; 0467; # CYRILLIC CAPITAL LETTER LITTLE YUS +0468; C; 0469; # CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS +046A; C; 046B; # CYRILLIC CAPITAL LETTER BIG YUS +046C; C; 046D; # CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS +046E; C; 046F; # CYRILLIC CAPITAL LETTER KSI +0470; C; 0471; # CYRILLIC CAPITAL LETTER PSI +0472; C; 0473; # CYRILLIC CAPITAL LETTER FITA +0474; C; 0475; # CYRILLIC CAPITAL LETTER IZHITSA +0476; C; 0477; # CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT +0478; C; 0479; # CYRILLIC CAPITAL LETTER UK +047A; C; 047B; # CYRILLIC CAPITAL LETTER ROUND OMEGA +047C; C; 047D; # CYRILLIC CAPITAL LETTER OMEGA WITH TITLO +047E; C; 047F; # CYRILLIC CAPITAL LETTER OT +0480; C; 0481; # CYRILLIC CAPITAL LETTER KOPPA +048A; C; 048B; # CYRILLIC CAPITAL LETTER SHORT I WITH TAIL +048C; C; 048D; # CYRILLIC CAPITAL LETTER SEMISOFT SIGN +048E; C; 048F; # CYRILLIC CAPITAL LETTER ER WITH TICK +0490; C; 0491; # CYRILLIC CAPITAL LETTER GHE WITH UPTURN +0492; C; 0493; # CYRILLIC CAPITAL LETTER GHE WITH STROKE +0494; C; 0495; # CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK +0496; C; 0497; # CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER +0498; C; 0499; # CYRILLIC CAPITAL LETTER ZE WITH DESCENDER +049A; C; 049B; # CYRILLIC CAPITAL LETTER KA WITH DESCENDER +049C; C; 049D; # CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE +049E; C; 049F; # CYRILLIC CAPITAL LETTER KA WITH STROKE +04A0; C; 04A1; # CYRILLIC CAPITAL LETTER BASHKIR KA +04A2; C; 04A3; # CYRILLIC CAPITAL LETTER EN WITH DESCENDER +04A4; C; 04A5; # CYRILLIC CAPITAL LIGATURE EN GHE +04A6; C; 04A7; # CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK +04A8; C; 04A9; # CYRILLIC CAPITAL LETTER ABKHASIAN HA +04AA; C; 04AB; # CYRILLIC CAPITAL LETTER ES WITH DESCENDER +04AC; C; 04AD; # CYRILLIC CAPITAL LETTER TE WITH DESCENDER +04AE; C; 04AF; # CYRILLIC CAPITAL LETTER STRAIGHT U +04B0; C; 04B1; # CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE +04B2; C; 04B3; # CYRILLIC CAPITAL LETTER HA WITH DESCENDER +04B4; C; 04B5; # CYRILLIC CAPITAL LIGATURE TE TSE +04B6; C; 04B7; # CYRILLIC CAPITAL LETTER CHE WITH DESCENDER +04B8; C; 04B9; # CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE +04BA; C; 04BB; # CYRILLIC CAPITAL LETTER SHHA +04BC; C; 04BD; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE +04BE; C; 04BF; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER +04C0; C; 04CF; # CYRILLIC LETTER PALOCHKA +04C1; C; 04C2; # CYRILLIC CAPITAL LETTER ZHE WITH BREVE +04C3; C; 04C4; # CYRILLIC CAPITAL LETTER KA WITH HOOK +04C5; C; 04C6; # CYRILLIC CAPITAL LETTER EL WITH TAIL +04C7; C; 04C8; # CYRILLIC CAPITAL LETTER EN WITH HOOK +04C9; C; 04CA; # CYRILLIC CAPITAL LETTER EN WITH TAIL +04CB; C; 04CC; # CYRILLIC CAPITAL LETTER KHAKASSIAN CHE +04CD; C; 04CE; # CYRILLIC CAPITAL LETTER EM WITH TAIL +04D0; C; 04D1; # CYRILLIC CAPITAL LETTER A WITH BREVE +04D2; C; 04D3; # CYRILLIC CAPITAL LETTER A WITH DIAERESIS +04D4; C; 04D5; # CYRILLIC CAPITAL LIGATURE A IE +04D6; C; 04D7; # CYRILLIC CAPITAL LETTER IE WITH BREVE +04D8; C; 04D9; # CYRILLIC CAPITAL LETTER SCHWA +04DA; C; 04DB; # CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS +04DC; C; 04DD; # CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS +04DE; C; 04DF; # CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS +04E0; C; 04E1; # CYRILLIC CAPITAL LETTER ABKHASIAN DZE +04E2; C; 04E3; # CYRILLIC CAPITAL LETTER I WITH MACRON +04E4; C; 04E5; # CYRILLIC CAPITAL LETTER I WITH DIAERESIS +04E6; C; 04E7; # CYRILLIC CAPITAL LETTER O WITH DIAERESIS +04E8; C; 04E9; # CYRILLIC CAPITAL LETTER BARRED O +04EA; C; 04EB; # CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS +04EC; C; 04ED; # CYRILLIC CAPITAL LETTER E WITH DIAERESIS +04EE; C; 04EF; # CYRILLIC CAPITAL LETTER U WITH MACRON +04F0; C; 04F1; # CYRILLIC CAPITAL LETTER U WITH DIAERESIS +04F2; C; 04F3; # CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE +04F4; C; 04F5; # CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS +04F6; C; 04F7; # CYRILLIC CAPITAL LETTER GHE WITH DESCENDER +04F8; C; 04F9; # CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS +04FA; C; 04FB; # CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK +04FC; C; 04FD; # CYRILLIC CAPITAL LETTER HA WITH HOOK +04FE; C; 04FF; # CYRILLIC CAPITAL LETTER HA WITH STROKE +0500; C; 0501; # CYRILLIC CAPITAL LETTER KOMI DE +0502; C; 0503; # CYRILLIC CAPITAL LETTER KOMI DJE +0504; C; 0505; # CYRILLIC CAPITAL LETTER KOMI ZJE +0506; C; 0507; # CYRILLIC CAPITAL LETTER KOMI DZJE +0508; C; 0509; # CYRILLIC CAPITAL LETTER KOMI LJE +050A; C; 050B; # CYRILLIC CAPITAL LETTER KOMI NJE +050C; C; 050D; # CYRILLIC CAPITAL LETTER KOMI SJE +050E; C; 050F; # CYRILLIC CAPITAL LETTER KOMI TJE +0510; C; 0511; # CYRILLIC CAPITAL LETTER REVERSED ZE +0512; C; 0513; # CYRILLIC CAPITAL LETTER EL WITH HOOK +0514; C; 0515; # CYRILLIC CAPITAL LETTER LHA +0516; C; 0517; # CYRILLIC CAPITAL LETTER RHA +0518; C; 0519; # CYRILLIC CAPITAL LETTER YAE +051A; C; 051B; # CYRILLIC CAPITAL LETTER QA +051C; C; 051D; # CYRILLIC CAPITAL LETTER WE +051E; C; 051F; # CYRILLIC CAPITAL LETTER ALEUT KA +0520; C; 0521; # CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK +0522; C; 0523; # CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK +0524; C; 0525; # CYRILLIC CAPITAL LETTER PE WITH DESCENDER +0526; C; 0527; # CYRILLIC CAPITAL LETTER SHHA WITH DESCENDER +0528; C; 0529; # CYRILLIC CAPITAL LETTER EN WITH LEFT HOOK +052A; C; 052B; # CYRILLIC CAPITAL LETTER DZZHE +052C; C; 052D; # CYRILLIC CAPITAL LETTER DCHE +052E; C; 052F; # CYRILLIC CAPITAL LETTER EL WITH DESCENDER +0531; C; 0561; # ARMENIAN CAPITAL LETTER AYB +0532; C; 0562; # ARMENIAN CAPITAL LETTER BEN +0533; C; 0563; # ARMENIAN CAPITAL LETTER GIM +0534; C; 0564; # ARMENIAN CAPITAL LETTER DA +0535; C; 0565; # ARMENIAN CAPITAL LETTER ECH +0536; C; 0566; # ARMENIAN CAPITAL LETTER ZA +0537; C; 0567; # ARMENIAN CAPITAL LETTER EH +0538; C; 0568; # ARMENIAN CAPITAL LETTER ET +0539; C; 0569; # ARMENIAN CAPITAL LETTER TO +053A; C; 056A; # ARMENIAN CAPITAL LETTER ZHE +053B; C; 056B; # ARMENIAN CAPITAL LETTER INI +053C; C; 056C; # ARMENIAN CAPITAL LETTER LIWN +053D; C; 056D; # ARMENIAN CAPITAL LETTER XEH +053E; C; 056E; # ARMENIAN CAPITAL LETTER CA +053F; C; 056F; # ARMENIAN CAPITAL LETTER KEN +0540; C; 0570; # ARMENIAN CAPITAL LETTER HO +0541; C; 0571; # ARMENIAN CAPITAL LETTER JA +0542; C; 0572; # ARMENIAN CAPITAL LETTER GHAD +0543; C; 0573; # ARMENIAN CAPITAL LETTER CHEH +0544; C; 0574; # ARMENIAN CAPITAL LETTER MEN +0545; C; 0575; # ARMENIAN CAPITAL LETTER YI +0546; C; 0576; # ARMENIAN CAPITAL LETTER NOW +0547; C; 0577; # ARMENIAN CAPITAL LETTER SHA +0548; C; 0578; # ARMENIAN CAPITAL LETTER VO +0549; C; 0579; # ARMENIAN CAPITAL LETTER CHA +054A; C; 057A; # ARMENIAN CAPITAL LETTER PEH +054B; C; 057B; # ARMENIAN CAPITAL LETTER JHEH +054C; C; 057C; # ARMENIAN CAPITAL LETTER RA +054D; C; 057D; # ARMENIAN CAPITAL LETTER SEH +054E; C; 057E; # ARMENIAN CAPITAL LETTER VEW +054F; C; 057F; # ARMENIAN CAPITAL LETTER TIWN +0550; C; 0580; # ARMENIAN CAPITAL LETTER REH +0551; C; 0581; # ARMENIAN CAPITAL LETTER CO +0552; C; 0582; # ARMENIAN CAPITAL LETTER YIWN +0553; C; 0583; # ARMENIAN CAPITAL LETTER PIWR +0554; C; 0584; # ARMENIAN CAPITAL LETTER KEH +0555; C; 0585; # ARMENIAN CAPITAL LETTER OH +0556; C; 0586; # ARMENIAN CAPITAL LETTER FEH +0587; F; 0565 0582; # ARMENIAN SMALL LIGATURE ECH YIWN +10A0; C; 2D00; # GEORGIAN CAPITAL LETTER AN +10A1; C; 2D01; # GEORGIAN CAPITAL LETTER BAN +10A2; C; 2D02; # GEORGIAN CAPITAL LETTER GAN +10A3; C; 2D03; # GEORGIAN CAPITAL LETTER DON +10A4; C; 2D04; # GEORGIAN CAPITAL LETTER EN +10A5; C; 2D05; # GEORGIAN CAPITAL LETTER VIN +10A6; C; 2D06; # GEORGIAN CAPITAL LETTER ZEN +10A7; C; 2D07; # GEORGIAN CAPITAL LETTER TAN +10A8; C; 2D08; # GEORGIAN CAPITAL LETTER IN +10A9; C; 2D09; # GEORGIAN CAPITAL LETTER KAN +10AA; C; 2D0A; # GEORGIAN CAPITAL LETTER LAS +10AB; C; 2D0B; # GEORGIAN CAPITAL LETTER MAN +10AC; C; 2D0C; # GEORGIAN CAPITAL LETTER NAR +10AD; C; 2D0D; # GEORGIAN CAPITAL LETTER ON +10AE; C; 2D0E; # GEORGIAN CAPITAL LETTER PAR +10AF; C; 2D0F; # GEORGIAN CAPITAL LETTER ZHAR +10B0; C; 2D10; # GEORGIAN CAPITAL LETTER RAE +10B1; C; 2D11; # GEORGIAN CAPITAL LETTER SAN +10B2; C; 2D12; # GEORGIAN CAPITAL LETTER TAR +10B3; C; 2D13; # GEORGIAN CAPITAL LETTER UN +10B4; C; 2D14; # GEORGIAN CAPITAL LETTER PHAR +10B5; C; 2D15; # GEORGIAN CAPITAL LETTER KHAR +10B6; C; 2D16; # GEORGIAN CAPITAL LETTER GHAN +10B7; C; 2D17; # GEORGIAN CAPITAL LETTER QAR +10B8; C; 2D18; # GEORGIAN CAPITAL LETTER SHIN +10B9; C; 2D19; # GEORGIAN CAPITAL LETTER CHIN +10BA; C; 2D1A; # GEORGIAN CAPITAL LETTER CAN +10BB; C; 2D1B; # GEORGIAN CAPITAL LETTER JIL +10BC; C; 2D1C; # GEORGIAN CAPITAL LETTER CIL +10BD; C; 2D1D; # GEORGIAN CAPITAL LETTER CHAR +10BE; C; 2D1E; # GEORGIAN CAPITAL LETTER XAN +10BF; C; 2D1F; # GEORGIAN CAPITAL LETTER JHAN +10C0; C; 2D20; # GEORGIAN CAPITAL LETTER HAE +10C1; C; 2D21; # GEORGIAN CAPITAL LETTER HE +10C2; C; 2D22; # GEORGIAN CAPITAL LETTER HIE +10C3; C; 2D23; # GEORGIAN CAPITAL LETTER WE +10C4; C; 2D24; # GEORGIAN CAPITAL LETTER HAR +10C5; C; 2D25; # GEORGIAN CAPITAL LETTER HOE +10C7; C; 2D27; # GEORGIAN CAPITAL LETTER YN +10CD; C; 2D2D; # GEORGIAN CAPITAL LETTER AEN +13F8; C; 13F0; # CHEROKEE SMALL LETTER YE +13F9; C; 13F1; # CHEROKEE SMALL LETTER YI +13FA; C; 13F2; # CHEROKEE SMALL LETTER YO +13FB; C; 13F3; # CHEROKEE SMALL LETTER YU +13FC; C; 13F4; # CHEROKEE SMALL LETTER YV +13FD; C; 13F5; # CHEROKEE SMALL LETTER MV +1C80; C; 0432; # CYRILLIC SMALL LETTER ROUNDED VE +1C81; C; 0434; # CYRILLIC SMALL LETTER LONG-LEGGED DE +1C82; C; 043E; # CYRILLIC SMALL LETTER NARROW O +1C83; C; 0441; # CYRILLIC SMALL LETTER WIDE ES +1C84; C; 0442; # CYRILLIC SMALL LETTER TALL TE +1C85; C; 0442; # CYRILLIC SMALL LETTER THREE-LEGGED TE +1C86; C; 044A; # CYRILLIC SMALL LETTER TALL HARD SIGN +1C87; C; 0463; # CYRILLIC SMALL LETTER TALL YAT +1C88; C; A64B; # CYRILLIC SMALL LETTER UNBLENDED UK +1E00; C; 1E01; # LATIN CAPITAL LETTER A WITH RING BELOW +1E02; C; 1E03; # LATIN CAPITAL LETTER B WITH DOT ABOVE +1E04; C; 1E05; # LATIN CAPITAL LETTER B WITH DOT BELOW +1E06; C; 1E07; # LATIN CAPITAL LETTER B WITH LINE BELOW +1E08; C; 1E09; # LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE +1E0A; C; 1E0B; # LATIN CAPITAL LETTER D WITH DOT ABOVE +1E0C; C; 1E0D; # LATIN CAPITAL LETTER D WITH DOT BELOW +1E0E; C; 1E0F; # LATIN CAPITAL LETTER D WITH LINE BELOW +1E10; C; 1E11; # LATIN CAPITAL LETTER D WITH CEDILLA +1E12; C; 1E13; # LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW +1E14; C; 1E15; # LATIN CAPITAL LETTER E WITH MACRON AND GRAVE +1E16; C; 1E17; # LATIN CAPITAL LETTER E WITH MACRON AND ACUTE +1E18; C; 1E19; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW +1E1A; C; 1E1B; # LATIN CAPITAL LETTER E WITH TILDE BELOW +1E1C; C; 1E1D; # LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE +1E1E; C; 1E1F; # LATIN CAPITAL LETTER F WITH DOT ABOVE +1E20; C; 1E21; # LATIN CAPITAL LETTER G WITH MACRON +1E22; C; 1E23; # LATIN CAPITAL LETTER H WITH DOT ABOVE +1E24; C; 1E25; # LATIN CAPITAL LETTER H WITH DOT BELOW +1E26; C; 1E27; # LATIN CAPITAL LETTER H WITH DIAERESIS +1E28; C; 1E29; # LATIN CAPITAL LETTER H WITH CEDILLA +1E2A; C; 1E2B; # LATIN CAPITAL LETTER H WITH BREVE BELOW +1E2C; C; 1E2D; # LATIN CAPITAL LETTER I WITH TILDE BELOW +1E2E; C; 1E2F; # LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE +1E30; C; 1E31; # LATIN CAPITAL LETTER K WITH ACUTE +1E32; C; 1E33; # LATIN CAPITAL LETTER K WITH DOT BELOW +1E34; C; 1E35; # LATIN CAPITAL LETTER K WITH LINE BELOW +1E36; C; 1E37; # LATIN CAPITAL LETTER L WITH DOT BELOW +1E38; C; 1E39; # LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON +1E3A; C; 1E3B; # LATIN CAPITAL LETTER L WITH LINE BELOW +1E3C; C; 1E3D; # LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW +1E3E; C; 1E3F; # LATIN CAPITAL LETTER M WITH ACUTE +1E40; C; 1E41; # LATIN CAPITAL LETTER M WITH DOT ABOVE +1E42; C; 1E43; # LATIN CAPITAL LETTER M WITH DOT BELOW +1E44; C; 1E45; # LATIN CAPITAL LETTER N WITH DOT ABOVE +1E46; C; 1E47; # LATIN CAPITAL LETTER N WITH DOT BELOW +1E48; C; 1E49; # LATIN CAPITAL LETTER N WITH LINE BELOW +1E4A; C; 1E4B; # LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW +1E4C; C; 1E4D; # LATIN CAPITAL LETTER O WITH TILDE AND ACUTE +1E4E; C; 1E4F; # LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS +1E50; C; 1E51; # LATIN CAPITAL LETTER O WITH MACRON AND GRAVE +1E52; C; 1E53; # LATIN CAPITAL LETTER O WITH MACRON AND ACUTE +1E54; C; 1E55; # LATIN CAPITAL LETTER P WITH ACUTE +1E56; C; 1E57; # LATIN CAPITAL LETTER P WITH DOT ABOVE +1E58; C; 1E59; # LATIN CAPITAL LETTER R WITH DOT ABOVE +1E5A; C; 1E5B; # LATIN CAPITAL LETTER R WITH DOT BELOW +1E5C; C; 1E5D; # LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON +1E5E; C; 1E5F; # LATIN CAPITAL LETTER R WITH LINE BELOW +1E60; C; 1E61; # LATIN CAPITAL LETTER S WITH DOT ABOVE +1E62; C; 1E63; # LATIN CAPITAL LETTER S WITH DOT BELOW +1E64; C; 1E65; # LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE +1E66; C; 1E67; # LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE +1E68; C; 1E69; # LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE +1E6A; C; 1E6B; # LATIN CAPITAL LETTER T WITH DOT ABOVE +1E6C; C; 1E6D; # LATIN CAPITAL LETTER T WITH DOT BELOW +1E6E; C; 1E6F; # LATIN CAPITAL LETTER T WITH LINE BELOW +1E70; C; 1E71; # LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW +1E72; C; 1E73; # LATIN CAPITAL LETTER U WITH DIAERESIS BELOW +1E74; C; 1E75; # LATIN CAPITAL LETTER U WITH TILDE BELOW +1E76; C; 1E77; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW +1E78; C; 1E79; # LATIN CAPITAL LETTER U WITH TILDE AND ACUTE +1E7A; C; 1E7B; # LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS +1E7C; C; 1E7D; # LATIN CAPITAL LETTER V WITH TILDE +1E7E; C; 1E7F; # LATIN CAPITAL LETTER V WITH DOT BELOW +1E80; C; 1E81; # LATIN CAPITAL LETTER W WITH GRAVE +1E82; C; 1E83; # LATIN CAPITAL LETTER W WITH ACUTE +1E84; C; 1E85; # LATIN CAPITAL LETTER W WITH DIAERESIS +1E86; C; 1E87; # LATIN CAPITAL LETTER W WITH DOT ABOVE +1E88; C; 1E89; # LATIN CAPITAL LETTER W WITH DOT BELOW +1E8A; C; 1E8B; # LATIN CAPITAL LETTER X WITH DOT ABOVE +1E8C; C; 1E8D; # LATIN CAPITAL LETTER X WITH DIAERESIS +1E8E; C; 1E8F; # LATIN CAPITAL LETTER Y WITH DOT ABOVE +1E90; C; 1E91; # LATIN CAPITAL LETTER Z WITH CIRCUMFLEX +1E92; C; 1E93; # LATIN CAPITAL LETTER Z WITH DOT BELOW +1E94; C; 1E95; # LATIN CAPITAL LETTER Z WITH LINE BELOW +1E96; F; 0068 0331; # LATIN SMALL LETTER H WITH LINE BELOW +1E97; F; 0074 0308; # LATIN SMALL LETTER T WITH DIAERESIS +1E98; F; 0077 030A; # LATIN SMALL LETTER W WITH RING ABOVE +1E99; F; 0079 030A; # LATIN SMALL LETTER Y WITH RING ABOVE +1E9A; F; 0061 02BE; # LATIN SMALL LETTER A WITH RIGHT HALF RING +1E9B; C; 1E61; # LATIN SMALL LETTER LONG S WITH DOT ABOVE +1E9E; F; 0073 0073; # LATIN CAPITAL LETTER SHARP S +1E9E; S; 00DF; # LATIN CAPITAL LETTER SHARP S +1EA0; C; 1EA1; # LATIN CAPITAL LETTER A WITH DOT BELOW +1EA2; C; 1EA3; # LATIN CAPITAL LETTER A WITH HOOK ABOVE +1EA4; C; 1EA5; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE +1EA6; C; 1EA7; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE +1EA8; C; 1EA9; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE +1EAA; C; 1EAB; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE +1EAC; C; 1EAD; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW +1EAE; C; 1EAF; # LATIN CAPITAL LETTER A WITH BREVE AND ACUTE +1EB0; C; 1EB1; # LATIN CAPITAL LETTER A WITH BREVE AND GRAVE +1EB2; C; 1EB3; # LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE +1EB4; C; 1EB5; # LATIN CAPITAL LETTER A WITH BREVE AND TILDE +1EB6; C; 1EB7; # LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW +1EB8; C; 1EB9; # LATIN CAPITAL LETTER E WITH DOT BELOW +1EBA; C; 1EBB; # LATIN CAPITAL LETTER E WITH HOOK ABOVE +1EBC; C; 1EBD; # LATIN CAPITAL LETTER E WITH TILDE +1EBE; C; 1EBF; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE +1EC0; C; 1EC1; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE +1EC2; C; 1EC3; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE +1EC4; C; 1EC5; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE +1EC6; C; 1EC7; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW +1EC8; C; 1EC9; # LATIN CAPITAL LETTER I WITH HOOK ABOVE +1ECA; C; 1ECB; # LATIN CAPITAL LETTER I WITH DOT BELOW +1ECC; C; 1ECD; # LATIN CAPITAL LETTER O WITH DOT BELOW +1ECE; C; 1ECF; # LATIN CAPITAL LETTER O WITH HOOK ABOVE +1ED0; C; 1ED1; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE +1ED2; C; 1ED3; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE +1ED4; C; 1ED5; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE +1ED6; C; 1ED7; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE +1ED8; C; 1ED9; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW +1EDA; C; 1EDB; # LATIN CAPITAL LETTER O WITH HORN AND ACUTE +1EDC; C; 1EDD; # LATIN CAPITAL LETTER O WITH HORN AND GRAVE +1EDE; C; 1EDF; # LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE +1EE0; C; 1EE1; # LATIN CAPITAL LETTER O WITH HORN AND TILDE +1EE2; C; 1EE3; # LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW +1EE4; C; 1EE5; # LATIN CAPITAL LETTER U WITH DOT BELOW +1EE6; C; 1EE7; # LATIN CAPITAL LETTER U WITH HOOK ABOVE +1EE8; C; 1EE9; # LATIN CAPITAL LETTER U WITH HORN AND ACUTE +1EEA; C; 1EEB; # LATIN CAPITAL LETTER U WITH HORN AND GRAVE +1EEC; C; 1EED; # LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE +1EEE; C; 1EEF; # LATIN CAPITAL LETTER U WITH HORN AND TILDE +1EF0; C; 1EF1; # LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW +1EF2; C; 1EF3; # LATIN CAPITAL LETTER Y WITH GRAVE +1EF4; C; 1EF5; # LATIN CAPITAL LETTER Y WITH DOT BELOW +1EF6; C; 1EF7; # LATIN CAPITAL LETTER Y WITH HOOK ABOVE +1EF8; C; 1EF9; # LATIN CAPITAL LETTER Y WITH TILDE +1EFA; C; 1EFB; # LATIN CAPITAL LETTER MIDDLE-WELSH LL +1EFC; C; 1EFD; # LATIN CAPITAL LETTER MIDDLE-WELSH V +1EFE; C; 1EFF; # LATIN CAPITAL LETTER Y WITH LOOP +1F08; C; 1F00; # GREEK CAPITAL LETTER ALPHA WITH PSILI +1F09; C; 1F01; # GREEK CAPITAL LETTER ALPHA WITH DASIA +1F0A; C; 1F02; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA +1F0B; C; 1F03; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA +1F0C; C; 1F04; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA +1F0D; C; 1F05; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA +1F0E; C; 1F06; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI +1F0F; C; 1F07; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI +1F18; C; 1F10; # GREEK CAPITAL LETTER EPSILON WITH PSILI +1F19; C; 1F11; # GREEK CAPITAL LETTER EPSILON WITH DASIA +1F1A; C; 1F12; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND VARIA +1F1B; C; 1F13; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND VARIA +1F1C; C; 1F14; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND OXIA +1F1D; C; 1F15; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA +1F28; C; 1F20; # GREEK CAPITAL LETTER ETA WITH PSILI +1F29; C; 1F21; # GREEK CAPITAL LETTER ETA WITH DASIA +1F2A; C; 1F22; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA +1F2B; C; 1F23; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA +1F2C; C; 1F24; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA +1F2D; C; 1F25; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA +1F2E; C; 1F26; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI +1F2F; C; 1F27; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI +1F38; C; 1F30; # GREEK CAPITAL LETTER IOTA WITH PSILI +1F39; C; 1F31; # GREEK CAPITAL LETTER IOTA WITH DASIA +1F3A; C; 1F32; # GREEK CAPITAL LETTER IOTA WITH PSILI AND VARIA +1F3B; C; 1F33; # GREEK CAPITAL LETTER IOTA WITH DASIA AND VARIA +1F3C; C; 1F34; # GREEK CAPITAL LETTER IOTA WITH PSILI AND OXIA +1F3D; C; 1F35; # GREEK CAPITAL LETTER IOTA WITH DASIA AND OXIA +1F3E; C; 1F36; # GREEK CAPITAL LETTER IOTA WITH PSILI AND PERISPOMENI +1F3F; C; 1F37; # GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI +1F48; C; 1F40; # GREEK CAPITAL LETTER OMICRON WITH PSILI +1F49; C; 1F41; # GREEK CAPITAL LETTER OMICRON WITH DASIA +1F4A; C; 1F42; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND VARIA +1F4B; C; 1F43; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND VARIA +1F4C; C; 1F44; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND OXIA +1F4D; C; 1F45; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA +1F50; F; 03C5 0313; # GREEK SMALL LETTER UPSILON WITH PSILI +1F52; F; 03C5 0313 0300; # GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA +1F54; F; 03C5 0313 0301; # GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA +1F56; F; 03C5 0313 0342; # GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI +1F59; C; 1F51; # GREEK CAPITAL LETTER UPSILON WITH DASIA +1F5B; C; 1F53; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA +1F5D; C; 1F55; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA +1F5F; C; 1F57; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F68; C; 1F60; # GREEK CAPITAL LETTER OMEGA WITH PSILI +1F69; C; 1F61; # GREEK CAPITAL LETTER OMEGA WITH DASIA +1F6A; C; 1F62; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA +1F6B; C; 1F63; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA +1F6C; C; 1F64; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA +1F6D; C; 1F65; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA +1F6E; C; 1F66; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI +1F6F; C; 1F67; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI +1F80; F; 1F00 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI +1F81; F; 1F01 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI +1F82; F; 1F02 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI +1F83; F; 1F03 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI +1F84; F; 1F04 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI +1F85; F; 1F05 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI +1F86; F; 1F06 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI +1F87; F; 1F07 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1F88; F; 1F00 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI +1F88; S; 1F80; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI +1F89; F; 1F01 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI +1F89; S; 1F81; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI +1F8A; F; 1F02 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1F8A; S; 1F82; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1F8B; F; 1F03 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1F8B; S; 1F83; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1F8C; F; 1F04 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1F8C; S; 1F84; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1F8D; F; 1F05 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1F8D; S; 1F85; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1F8E; F; 1F06 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1F8E; S; 1F86; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1F8F; F; 1F07 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1F8F; S; 1F87; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1F90; F; 1F20 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI +1F91; F; 1F21 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI +1F92; F; 1F22 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI +1F93; F; 1F23 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI +1F94; F; 1F24 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI +1F95; F; 1F25 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI +1F96; F; 1F26 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI +1F97; F; 1F27 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1F98; F; 1F20 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI +1F98; S; 1F90; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI +1F99; F; 1F21 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI +1F99; S; 1F91; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI +1F9A; F; 1F22 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1F9A; S; 1F92; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1F9B; F; 1F23 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1F9B; S; 1F93; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1F9C; F; 1F24 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1F9C; S; 1F94; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1F9D; F; 1F25 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1F9D; S; 1F95; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1F9E; F; 1F26 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1F9E; S; 1F96; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1F9F; F; 1F27 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1F9F; S; 1F97; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1FA0; F; 1F60 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI +1FA1; F; 1F61 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI +1FA2; F; 1F62 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI +1FA3; F; 1F63 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI +1FA4; F; 1F64 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI +1FA5; F; 1F65 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI +1FA6; F; 1F66 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI +1FA7; F; 1F67 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1FA8; F; 1F60 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI +1FA8; S; 1FA0; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI +1FA9; F; 1F61 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI +1FA9; S; 1FA1; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI +1FAA; F; 1F62 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1FAA; S; 1FA2; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1FAB; F; 1F63 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1FAB; S; 1FA3; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1FAC; F; 1F64 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1FAC; S; 1FA4; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1FAD; F; 1F65 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1FAD; S; 1FA5; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1FAE; F; 1F66 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1FAE; S; 1FA6; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1FAF; F; 1F67 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1FAF; S; 1FA7; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1FB2; F; 1F70 03B9; # GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI +1FB3; F; 03B1 03B9; # GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI +1FB4; F; 03AC 03B9; # GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FB6; F; 03B1 0342; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI +1FB7; F; 03B1 0342 03B9; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI +1FB8; C; 1FB0; # GREEK CAPITAL LETTER ALPHA WITH VRACHY +1FB9; C; 1FB1; # GREEK CAPITAL LETTER ALPHA WITH MACRON +1FBA; C; 1F70; # GREEK CAPITAL LETTER ALPHA WITH VARIA +1FBB; C; 1F71; # GREEK CAPITAL LETTER ALPHA WITH OXIA +1FBC; F; 03B1 03B9; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FBC; S; 1FB3; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FBE; C; 03B9; # GREEK PROSGEGRAMMENI +1FC2; F; 1F74 03B9; # GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI +1FC3; F; 03B7 03B9; # GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI +1FC4; F; 03AE 03B9; # GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC6; F; 03B7 0342; # GREEK SMALL LETTER ETA WITH PERISPOMENI +1FC7; F; 03B7 0342 03B9; # GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI +1FC8; C; 1F72; # GREEK CAPITAL LETTER EPSILON WITH VARIA +1FC9; C; 1F73; # GREEK CAPITAL LETTER EPSILON WITH OXIA +1FCA; C; 1F74; # GREEK CAPITAL LETTER ETA WITH VARIA +1FCB; C; 1F75; # GREEK CAPITAL LETTER ETA WITH OXIA +1FCC; F; 03B7 03B9; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FCC; S; 1FC3; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FD2; F; 03B9 0308 0300; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA +1FD3; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD6; F; 03B9 0342; # GREEK SMALL LETTER IOTA WITH PERISPOMENI +1FD7; F; 03B9 0308 0342; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI +1FD8; C; 1FD0; # GREEK CAPITAL LETTER IOTA WITH VRACHY +1FD9; C; 1FD1; # GREEK CAPITAL LETTER IOTA WITH MACRON +1FDA; C; 1F76; # GREEK CAPITAL LETTER IOTA WITH VARIA +1FDB; C; 1F77; # GREEK CAPITAL LETTER IOTA WITH OXIA +1FE2; F; 03C5 0308 0300; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA +1FE3; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA +1FE4; F; 03C1 0313; # GREEK SMALL LETTER RHO WITH PSILI +1FE6; F; 03C5 0342; # GREEK SMALL LETTER UPSILON WITH PERISPOMENI +1FE7; F; 03C5 0308 0342; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI +1FE8; C; 1FE0; # GREEK CAPITAL LETTER UPSILON WITH VRACHY +1FE9; C; 1FE1; # GREEK CAPITAL LETTER UPSILON WITH MACRON +1FEA; C; 1F7A; # GREEK CAPITAL LETTER UPSILON WITH VARIA +1FEB; C; 1F7B; # GREEK CAPITAL LETTER UPSILON WITH OXIA +1FEC; C; 1FE5; # GREEK CAPITAL LETTER RHO WITH DASIA +1FF2; F; 1F7C 03B9; # GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI +1FF3; F; 03C9 03B9; # GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI +1FF4; F; 03CE 03B9; # GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF6; F; 03C9 0342; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI +1FF7; F; 03C9 0342 03B9; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI +1FF8; C; 1F78; # GREEK CAPITAL LETTER OMICRON WITH VARIA +1FF9; C; 1F79; # GREEK CAPITAL LETTER OMICRON WITH OXIA +1FFA; C; 1F7C; # GREEK CAPITAL LETTER OMEGA WITH VARIA +1FFB; C; 1F7D; # GREEK CAPITAL LETTER OMEGA WITH OXIA +1FFC; F; 03C9 03B9; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +1FFC; S; 1FF3; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +2126; C; 03C9; # OHM SIGN +212A; C; 006B; # KELVIN SIGN +212B; C; 00E5; # ANGSTROM SIGN +2132; C; 214E; # TURNED CAPITAL F +2160; C; 2170; # ROMAN NUMERAL ONE +2161; C; 2171; # ROMAN NUMERAL TWO +2162; C; 2172; # ROMAN NUMERAL THREE +2163; C; 2173; # ROMAN NUMERAL FOUR +2164; C; 2174; # ROMAN NUMERAL FIVE +2165; C; 2175; # ROMAN NUMERAL SIX +2166; C; 2176; # ROMAN NUMERAL SEVEN +2167; C; 2177; # ROMAN NUMERAL EIGHT +2168; C; 2178; # ROMAN NUMERAL NINE +2169; C; 2179; # ROMAN NUMERAL TEN +216A; C; 217A; # ROMAN NUMERAL ELEVEN +216B; C; 217B; # ROMAN NUMERAL TWELVE +216C; C; 217C; # ROMAN NUMERAL FIFTY +216D; C; 217D; # ROMAN NUMERAL ONE HUNDRED +216E; C; 217E; # ROMAN NUMERAL FIVE HUNDRED +216F; C; 217F; # ROMAN NUMERAL ONE THOUSAND +2183; C; 2184; # ROMAN NUMERAL REVERSED ONE HUNDRED +24B6; C; 24D0; # CIRCLED LATIN CAPITAL LETTER A +24B7; C; 24D1; # CIRCLED LATIN CAPITAL LETTER B +24B8; C; 24D2; # CIRCLED LATIN CAPITAL LETTER C +24B9; C; 24D3; # CIRCLED LATIN CAPITAL LETTER D +24BA; C; 24D4; # CIRCLED LATIN CAPITAL LETTER E +24BB; C; 24D5; # CIRCLED LATIN CAPITAL LETTER F +24BC; C; 24D6; # CIRCLED LATIN CAPITAL LETTER G +24BD; C; 24D7; # CIRCLED LATIN CAPITAL LETTER H +24BE; C; 24D8; # CIRCLED LATIN CAPITAL LETTER I +24BF; C; 24D9; # CIRCLED LATIN CAPITAL LETTER J +24C0; C; 24DA; # CIRCLED LATIN CAPITAL LETTER K +24C1; C; 24DB; # CIRCLED LATIN CAPITAL LETTER L +24C2; C; 24DC; # CIRCLED LATIN CAPITAL LETTER M +24C3; C; 24DD; # CIRCLED LATIN CAPITAL LETTER N +24C4; C; 24DE; # CIRCLED LATIN CAPITAL LETTER O +24C5; C; 24DF; # CIRCLED LATIN CAPITAL LETTER P +24C6; C; 24E0; # CIRCLED LATIN CAPITAL LETTER Q +24C7; C; 24E1; # CIRCLED LATIN CAPITAL LETTER R +24C8; C; 24E2; # CIRCLED LATIN CAPITAL LETTER S +24C9; C; 24E3; # CIRCLED LATIN CAPITAL LETTER T +24CA; C; 24E4; # CIRCLED LATIN CAPITAL LETTER U +24CB; C; 24E5; # CIRCLED LATIN CAPITAL LETTER V +24CC; C; 24E6; # CIRCLED LATIN CAPITAL LETTER W +24CD; C; 24E7; # CIRCLED LATIN CAPITAL LETTER X +24CE; C; 24E8; # CIRCLED LATIN CAPITAL LETTER Y +24CF; C; 24E9; # CIRCLED LATIN CAPITAL LETTER Z +2C00; C; 2C30; # GLAGOLITIC CAPITAL LETTER AZU +2C01; C; 2C31; # GLAGOLITIC CAPITAL LETTER BUKY +2C02; C; 2C32; # GLAGOLITIC CAPITAL LETTER VEDE +2C03; C; 2C33; # GLAGOLITIC CAPITAL LETTER GLAGOLI +2C04; C; 2C34; # GLAGOLITIC CAPITAL LETTER DOBRO +2C05; C; 2C35; # GLAGOLITIC CAPITAL LETTER YESTU +2C06; C; 2C36; # GLAGOLITIC CAPITAL LETTER ZHIVETE +2C07; C; 2C37; # GLAGOLITIC CAPITAL LETTER DZELO +2C08; C; 2C38; # GLAGOLITIC CAPITAL LETTER ZEMLJA +2C09; C; 2C39; # GLAGOLITIC CAPITAL LETTER IZHE +2C0A; C; 2C3A; # GLAGOLITIC CAPITAL LETTER INITIAL IZHE +2C0B; C; 2C3B; # GLAGOLITIC CAPITAL LETTER I +2C0C; C; 2C3C; # GLAGOLITIC CAPITAL LETTER DJERVI +2C0D; C; 2C3D; # GLAGOLITIC CAPITAL LETTER KAKO +2C0E; C; 2C3E; # GLAGOLITIC CAPITAL LETTER LJUDIJE +2C0F; C; 2C3F; # GLAGOLITIC CAPITAL LETTER MYSLITE +2C10; C; 2C40; # GLAGOLITIC CAPITAL LETTER NASHI +2C11; C; 2C41; # GLAGOLITIC CAPITAL LETTER ONU +2C12; C; 2C42; # GLAGOLITIC CAPITAL LETTER POKOJI +2C13; C; 2C43; # GLAGOLITIC CAPITAL LETTER RITSI +2C14; C; 2C44; # GLAGOLITIC CAPITAL LETTER SLOVO +2C15; C; 2C45; # GLAGOLITIC CAPITAL LETTER TVRIDO +2C16; C; 2C46; # GLAGOLITIC CAPITAL LETTER UKU +2C17; C; 2C47; # GLAGOLITIC CAPITAL LETTER FRITU +2C18; C; 2C48; # GLAGOLITIC CAPITAL LETTER HERU +2C19; C; 2C49; # GLAGOLITIC CAPITAL LETTER OTU +2C1A; C; 2C4A; # GLAGOLITIC CAPITAL LETTER PE +2C1B; C; 2C4B; # GLAGOLITIC CAPITAL LETTER SHTA +2C1C; C; 2C4C; # GLAGOLITIC CAPITAL LETTER TSI +2C1D; C; 2C4D; # GLAGOLITIC CAPITAL LETTER CHRIVI +2C1E; C; 2C4E; # GLAGOLITIC CAPITAL LETTER SHA +2C1F; C; 2C4F; # GLAGOLITIC CAPITAL LETTER YERU +2C20; C; 2C50; # GLAGOLITIC CAPITAL LETTER YERI +2C21; C; 2C51; # GLAGOLITIC CAPITAL LETTER YATI +2C22; C; 2C52; # GLAGOLITIC CAPITAL LETTER SPIDERY HA +2C23; C; 2C53; # GLAGOLITIC CAPITAL LETTER YU +2C24; C; 2C54; # GLAGOLITIC CAPITAL LETTER SMALL YUS +2C25; C; 2C55; # GLAGOLITIC CAPITAL LETTER SMALL YUS WITH TAIL +2C26; C; 2C56; # GLAGOLITIC CAPITAL LETTER YO +2C27; C; 2C57; # GLAGOLITIC CAPITAL LETTER IOTATED SMALL YUS +2C28; C; 2C58; # GLAGOLITIC CAPITAL LETTER BIG YUS +2C29; C; 2C59; # GLAGOLITIC CAPITAL LETTER IOTATED BIG YUS +2C2A; C; 2C5A; # GLAGOLITIC CAPITAL LETTER FITA +2C2B; C; 2C5B; # GLAGOLITIC CAPITAL LETTER IZHITSA +2C2C; C; 2C5C; # GLAGOLITIC CAPITAL LETTER SHTAPIC +2C2D; C; 2C5D; # GLAGOLITIC CAPITAL LETTER TROKUTASTI A +2C2E; C; 2C5E; # GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE +2C60; C; 2C61; # LATIN CAPITAL LETTER L WITH DOUBLE BAR +2C62; C; 026B; # LATIN CAPITAL LETTER L WITH MIDDLE TILDE +2C63; C; 1D7D; # LATIN CAPITAL LETTER P WITH STROKE +2C64; C; 027D; # LATIN CAPITAL LETTER R WITH TAIL +2C67; C; 2C68; # LATIN CAPITAL LETTER H WITH DESCENDER +2C69; C; 2C6A; # LATIN CAPITAL LETTER K WITH DESCENDER +2C6B; C; 2C6C; # LATIN CAPITAL LETTER Z WITH DESCENDER +2C6D; C; 0251; # LATIN CAPITAL LETTER ALPHA +2C6E; C; 0271; # LATIN CAPITAL LETTER M WITH HOOK +2C6F; C; 0250; # LATIN CAPITAL LETTER TURNED A +2C70; C; 0252; # LATIN CAPITAL LETTER TURNED ALPHA +2C72; C; 2C73; # LATIN CAPITAL LETTER W WITH HOOK +2C75; C; 2C76; # LATIN CAPITAL LETTER HALF H +2C7E; C; 023F; # LATIN CAPITAL LETTER S WITH SWASH TAIL +2C7F; C; 0240; # LATIN CAPITAL LETTER Z WITH SWASH TAIL +2C80; C; 2C81; # COPTIC CAPITAL LETTER ALFA +2C82; C; 2C83; # COPTIC CAPITAL LETTER VIDA +2C84; C; 2C85; # COPTIC CAPITAL LETTER GAMMA +2C86; C; 2C87; # COPTIC CAPITAL LETTER DALDA +2C88; C; 2C89; # COPTIC CAPITAL LETTER EIE +2C8A; C; 2C8B; # COPTIC CAPITAL LETTER SOU +2C8C; C; 2C8D; # COPTIC CAPITAL LETTER ZATA +2C8E; C; 2C8F; # COPTIC CAPITAL LETTER HATE +2C90; C; 2C91; # COPTIC CAPITAL LETTER THETHE +2C92; C; 2C93; # COPTIC CAPITAL LETTER IAUDA +2C94; C; 2C95; # COPTIC CAPITAL LETTER KAPA +2C96; C; 2C97; # COPTIC CAPITAL LETTER LAULA +2C98; C; 2C99; # COPTIC CAPITAL LETTER MI +2C9A; C; 2C9B; # COPTIC CAPITAL LETTER NI +2C9C; C; 2C9D; # COPTIC CAPITAL LETTER KSI +2C9E; C; 2C9F; # COPTIC CAPITAL LETTER O +2CA0; C; 2CA1; # COPTIC CAPITAL LETTER PI +2CA2; C; 2CA3; # COPTIC CAPITAL LETTER RO +2CA4; C; 2CA5; # COPTIC CAPITAL LETTER SIMA +2CA6; C; 2CA7; # COPTIC CAPITAL LETTER TAU +2CA8; C; 2CA9; # COPTIC CAPITAL LETTER UA +2CAA; C; 2CAB; # COPTIC CAPITAL LETTER FI +2CAC; C; 2CAD; # COPTIC CAPITAL LETTER KHI +2CAE; C; 2CAF; # COPTIC CAPITAL LETTER PSI +2CB0; C; 2CB1; # COPTIC CAPITAL LETTER OOU +2CB2; C; 2CB3; # COPTIC CAPITAL LETTER DIALECT-P ALEF +2CB4; C; 2CB5; # COPTIC CAPITAL LETTER OLD COPTIC AIN +2CB6; C; 2CB7; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC EIE +2CB8; C; 2CB9; # COPTIC CAPITAL LETTER DIALECT-P KAPA +2CBA; C; 2CBB; # COPTIC CAPITAL LETTER DIALECT-P NI +2CBC; C; 2CBD; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC NI +2CBE; C; 2CBF; # COPTIC CAPITAL LETTER OLD COPTIC OOU +2CC0; C; 2CC1; # COPTIC CAPITAL LETTER SAMPI +2CC2; C; 2CC3; # COPTIC CAPITAL LETTER CROSSED SHEI +2CC4; C; 2CC5; # COPTIC CAPITAL LETTER OLD COPTIC SHEI +2CC6; C; 2CC7; # COPTIC CAPITAL LETTER OLD COPTIC ESH +2CC8; C; 2CC9; # COPTIC CAPITAL LETTER AKHMIMIC KHEI +2CCA; C; 2CCB; # COPTIC CAPITAL LETTER DIALECT-P HORI +2CCC; C; 2CCD; # COPTIC CAPITAL LETTER OLD COPTIC HORI +2CCE; C; 2CCF; # COPTIC CAPITAL LETTER OLD COPTIC HA +2CD0; C; 2CD1; # COPTIC CAPITAL LETTER L-SHAPED HA +2CD2; C; 2CD3; # COPTIC CAPITAL LETTER OLD COPTIC HEI +2CD4; C; 2CD5; # COPTIC CAPITAL LETTER OLD COPTIC HAT +2CD6; C; 2CD7; # COPTIC CAPITAL LETTER OLD COPTIC GANGIA +2CD8; C; 2CD9; # COPTIC CAPITAL LETTER OLD COPTIC DJA +2CDA; C; 2CDB; # COPTIC CAPITAL LETTER OLD COPTIC SHIMA +2CDC; C; 2CDD; # COPTIC CAPITAL LETTER OLD NUBIAN SHIMA +2CDE; C; 2CDF; # COPTIC CAPITAL LETTER OLD NUBIAN NGI +2CE0; C; 2CE1; # COPTIC CAPITAL LETTER OLD NUBIAN NYI +2CE2; C; 2CE3; # COPTIC CAPITAL LETTER OLD NUBIAN WAU +2CEB; C; 2CEC; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI +2CED; C; 2CEE; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC GANGIA +2CF2; C; 2CF3; # COPTIC CAPITAL LETTER BOHAIRIC KHEI +A640; C; A641; # CYRILLIC CAPITAL LETTER ZEMLYA +A642; C; A643; # CYRILLIC CAPITAL LETTER DZELO +A644; C; A645; # CYRILLIC CAPITAL LETTER REVERSED DZE +A646; C; A647; # CYRILLIC CAPITAL LETTER IOTA +A648; C; A649; # CYRILLIC CAPITAL LETTER DJERV +A64A; C; A64B; # CYRILLIC CAPITAL LETTER MONOGRAPH UK +A64C; C; A64D; # CYRILLIC CAPITAL LETTER BROAD OMEGA +A64E; C; A64F; # CYRILLIC CAPITAL LETTER NEUTRAL YER +A650; C; A651; # CYRILLIC CAPITAL LETTER YERU WITH BACK YER +A652; C; A653; # CYRILLIC CAPITAL LETTER IOTIFIED YAT +A654; C; A655; # CYRILLIC CAPITAL LETTER REVERSED YU +A656; C; A657; # CYRILLIC CAPITAL LETTER IOTIFIED A +A658; C; A659; # CYRILLIC CAPITAL LETTER CLOSED LITTLE YUS +A65A; C; A65B; # CYRILLIC CAPITAL LETTER BLENDED YUS +A65C; C; A65D; # CYRILLIC CAPITAL LETTER IOTIFIED CLOSED LITTLE YUS +A65E; C; A65F; # CYRILLIC CAPITAL LETTER YN +A660; C; A661; # CYRILLIC CAPITAL LETTER REVERSED TSE +A662; C; A663; # CYRILLIC CAPITAL LETTER SOFT DE +A664; C; A665; # CYRILLIC CAPITAL LETTER SOFT EL +A666; C; A667; # CYRILLIC CAPITAL LETTER SOFT EM +A668; C; A669; # CYRILLIC CAPITAL LETTER MONOCULAR O +A66A; C; A66B; # CYRILLIC CAPITAL LETTER BINOCULAR O +A66C; C; A66D; # CYRILLIC CAPITAL LETTER DOUBLE MONOCULAR O +A680; C; A681; # CYRILLIC CAPITAL LETTER DWE +A682; C; A683; # CYRILLIC CAPITAL LETTER DZWE +A684; C; A685; # CYRILLIC CAPITAL LETTER ZHWE +A686; C; A687; # CYRILLIC CAPITAL LETTER CCHE +A688; C; A689; # CYRILLIC CAPITAL LETTER DZZE +A68A; C; A68B; # CYRILLIC CAPITAL LETTER TE WITH MIDDLE HOOK +A68C; C; A68D; # CYRILLIC CAPITAL LETTER TWE +A68E; C; A68F; # CYRILLIC CAPITAL LETTER TSWE +A690; C; A691; # CYRILLIC CAPITAL LETTER TSSE +A692; C; A693; # CYRILLIC CAPITAL LETTER TCHE +A694; C; A695; # CYRILLIC CAPITAL LETTER HWE +A696; C; A697; # CYRILLIC CAPITAL LETTER SHWE +A698; C; A699; # CYRILLIC CAPITAL LETTER DOUBLE O +A69A; C; A69B; # CYRILLIC CAPITAL LETTER CROSSED O +A722; C; A723; # LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF +A724; C; A725; # LATIN CAPITAL LETTER EGYPTOLOGICAL AIN +A726; C; A727; # LATIN CAPITAL LETTER HENG +A728; C; A729; # LATIN CAPITAL LETTER TZ +A72A; C; A72B; # LATIN CAPITAL LETTER TRESILLO +A72C; C; A72D; # LATIN CAPITAL LETTER CUATRILLO +A72E; C; A72F; # LATIN CAPITAL LETTER CUATRILLO WITH COMMA +A732; C; A733; # LATIN CAPITAL LETTER AA +A734; C; A735; # LATIN CAPITAL LETTER AO +A736; C; A737; # LATIN CAPITAL LETTER AU +A738; C; A739; # LATIN CAPITAL LETTER AV +A73A; C; A73B; # LATIN CAPITAL LETTER AV WITH HORIZONTAL BAR +A73C; C; A73D; # LATIN CAPITAL LETTER AY +A73E; C; A73F; # LATIN CAPITAL LETTER REVERSED C WITH DOT +A740; C; A741; # LATIN CAPITAL LETTER K WITH STROKE +A742; C; A743; # LATIN CAPITAL LETTER K WITH DIAGONAL STROKE +A744; C; A745; # LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE +A746; C; A747; # LATIN CAPITAL LETTER BROKEN L +A748; C; A749; # LATIN CAPITAL LETTER L WITH HIGH STROKE +A74A; C; A74B; # LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY +A74C; C; A74D; # LATIN CAPITAL LETTER O WITH LOOP +A74E; C; A74F; # LATIN CAPITAL LETTER OO +A750; C; A751; # LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER +A752; C; A753; # LATIN CAPITAL LETTER P WITH FLOURISH +A754; C; A755; # LATIN CAPITAL LETTER P WITH SQUIRREL TAIL +A756; C; A757; # LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER +A758; C; A759; # LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE +A75A; C; A75B; # LATIN CAPITAL LETTER R ROTUNDA +A75C; C; A75D; # LATIN CAPITAL LETTER RUM ROTUNDA +A75E; C; A75F; # LATIN CAPITAL LETTER V WITH DIAGONAL STROKE +A760; C; A761; # LATIN CAPITAL LETTER VY +A762; C; A763; # LATIN CAPITAL LETTER VISIGOTHIC Z +A764; C; A765; # LATIN CAPITAL LETTER THORN WITH STROKE +A766; C; A767; # LATIN CAPITAL LETTER THORN WITH STROKE THROUGH DESCENDER +A768; C; A769; # LATIN CAPITAL LETTER VEND +A76A; C; A76B; # LATIN CAPITAL LETTER ET +A76C; C; A76D; # LATIN CAPITAL LETTER IS +A76E; C; A76F; # LATIN CAPITAL LETTER CON +A779; C; A77A; # LATIN CAPITAL LETTER INSULAR D +A77B; C; A77C; # LATIN CAPITAL LETTER INSULAR F +A77D; C; 1D79; # LATIN CAPITAL LETTER INSULAR G +A77E; C; A77F; # LATIN CAPITAL LETTER TURNED INSULAR G +A780; C; A781; # LATIN CAPITAL LETTER TURNED L +A782; C; A783; # LATIN CAPITAL LETTER INSULAR R +A784; C; A785; # LATIN CAPITAL LETTER INSULAR S +A786; C; A787; # LATIN CAPITAL LETTER INSULAR T +A78B; C; A78C; # LATIN CAPITAL LETTER SALTILLO +A78D; C; 0265; # LATIN CAPITAL LETTER TURNED H +A790; C; A791; # LATIN CAPITAL LETTER N WITH DESCENDER +A792; C; A793; # LATIN CAPITAL LETTER C WITH BAR +A796; C; A797; # LATIN CAPITAL LETTER B WITH FLOURISH +A798; C; A799; # LATIN CAPITAL LETTER F WITH STROKE +A79A; C; A79B; # LATIN CAPITAL LETTER VOLAPUK AE +A79C; C; A79D; # LATIN CAPITAL LETTER VOLAPUK OE +A79E; C; A79F; # LATIN CAPITAL LETTER VOLAPUK UE +A7A0; C; A7A1; # LATIN CAPITAL LETTER G WITH OBLIQUE STROKE +A7A2; C; A7A3; # LATIN CAPITAL LETTER K WITH OBLIQUE STROKE +A7A4; C; A7A5; # LATIN CAPITAL LETTER N WITH OBLIQUE STROKE +A7A6; C; A7A7; # LATIN CAPITAL LETTER R WITH OBLIQUE STROKE +A7A8; C; A7A9; # LATIN CAPITAL LETTER S WITH OBLIQUE STROKE +A7AA; C; 0266; # LATIN CAPITAL LETTER H WITH HOOK +A7AB; C; 025C; # LATIN CAPITAL LETTER REVERSED OPEN E +A7AC; C; 0261; # LATIN CAPITAL LETTER SCRIPT G +A7AD; C; 026C; # LATIN CAPITAL LETTER L WITH BELT +A7AE; C; 026A; # LATIN CAPITAL LETTER SMALL CAPITAL I +A7B0; C; 029E; # LATIN CAPITAL LETTER TURNED K +A7B1; C; 0287; # LATIN CAPITAL LETTER TURNED T +A7B2; C; 029D; # LATIN CAPITAL LETTER J WITH CROSSED-TAIL +A7B3; C; AB53; # LATIN CAPITAL LETTER CHI +A7B4; C; A7B5; # LATIN CAPITAL LETTER BETA +A7B6; C; A7B7; # LATIN CAPITAL LETTER OMEGA +AB70; C; 13A0; # CHEROKEE SMALL LETTER A +AB71; C; 13A1; # CHEROKEE SMALL LETTER E +AB72; C; 13A2; # CHEROKEE SMALL LETTER I +AB73; C; 13A3; # CHEROKEE SMALL LETTER O +AB74; C; 13A4; # CHEROKEE SMALL LETTER U +AB75; C; 13A5; # CHEROKEE SMALL LETTER V +AB76; C; 13A6; # CHEROKEE SMALL LETTER GA +AB77; C; 13A7; # CHEROKEE SMALL LETTER KA +AB78; C; 13A8; # CHEROKEE SMALL LETTER GE +AB79; C; 13A9; # CHEROKEE SMALL LETTER GI +AB7A; C; 13AA; # CHEROKEE SMALL LETTER GO +AB7B; C; 13AB; # CHEROKEE SMALL LETTER GU +AB7C; C; 13AC; # CHEROKEE SMALL LETTER GV +AB7D; C; 13AD; # CHEROKEE SMALL LETTER HA +AB7E; C; 13AE; # CHEROKEE SMALL LETTER HE +AB7F; C; 13AF; # CHEROKEE SMALL LETTER HI +AB80; C; 13B0; # CHEROKEE SMALL LETTER HO +AB81; C; 13B1; # CHEROKEE SMALL LETTER HU +AB82; C; 13B2; # CHEROKEE SMALL LETTER HV +AB83; C; 13B3; # CHEROKEE SMALL LETTER LA +AB84; C; 13B4; # CHEROKEE SMALL LETTER LE +AB85; C; 13B5; # CHEROKEE SMALL LETTER LI +AB86; C; 13B6; # CHEROKEE SMALL LETTER LO +AB87; C; 13B7; # CHEROKEE SMALL LETTER LU +AB88; C; 13B8; # CHEROKEE SMALL LETTER LV +AB89; C; 13B9; # CHEROKEE SMALL LETTER MA +AB8A; C; 13BA; # CHEROKEE SMALL LETTER ME +AB8B; C; 13BB; # CHEROKEE SMALL LETTER MI +AB8C; C; 13BC; # CHEROKEE SMALL LETTER MO +AB8D; C; 13BD; # CHEROKEE SMALL LETTER MU +AB8E; C; 13BE; # CHEROKEE SMALL LETTER NA +AB8F; C; 13BF; # CHEROKEE SMALL LETTER HNA +AB90; C; 13C0; # CHEROKEE SMALL LETTER NAH +AB91; C; 13C1; # CHEROKEE SMALL LETTER NE +AB92; C; 13C2; # CHEROKEE SMALL LETTER NI +AB93; C; 13C3; # CHEROKEE SMALL LETTER NO +AB94; C; 13C4; # CHEROKEE SMALL LETTER NU +AB95; C; 13C5; # CHEROKEE SMALL LETTER NV +AB96; C; 13C6; # CHEROKEE SMALL LETTER QUA +AB97; C; 13C7; # CHEROKEE SMALL LETTER QUE +AB98; C; 13C8; # CHEROKEE SMALL LETTER QUI +AB99; C; 13C9; # CHEROKEE SMALL LETTER QUO +AB9A; C; 13CA; # CHEROKEE SMALL LETTER QUU +AB9B; C; 13CB; # CHEROKEE SMALL LETTER QUV +AB9C; C; 13CC; # CHEROKEE SMALL LETTER SA +AB9D; C; 13CD; # CHEROKEE SMALL LETTER S +AB9E; C; 13CE; # CHEROKEE SMALL LETTER SE +AB9F; C; 13CF; # CHEROKEE SMALL LETTER SI +ABA0; C; 13D0; # CHEROKEE SMALL LETTER SO +ABA1; C; 13D1; # CHEROKEE SMALL LETTER SU +ABA2; C; 13D2; # CHEROKEE SMALL LETTER SV +ABA3; C; 13D3; # CHEROKEE SMALL LETTER DA +ABA4; C; 13D4; # CHEROKEE SMALL LETTER TA +ABA5; C; 13D5; # CHEROKEE SMALL LETTER DE +ABA6; C; 13D6; # CHEROKEE SMALL LETTER TE +ABA7; C; 13D7; # CHEROKEE SMALL LETTER DI +ABA8; C; 13D8; # CHEROKEE SMALL LETTER TI +ABA9; C; 13D9; # CHEROKEE SMALL LETTER DO +ABAA; C; 13DA; # CHEROKEE SMALL LETTER DU +ABAB; C; 13DB; # CHEROKEE SMALL LETTER DV +ABAC; C; 13DC; # CHEROKEE SMALL LETTER DLA +ABAD; C; 13DD; # CHEROKEE SMALL LETTER TLA +ABAE; C; 13DE; # CHEROKEE SMALL LETTER TLE +ABAF; C; 13DF; # CHEROKEE SMALL LETTER TLI +ABB0; C; 13E0; # CHEROKEE SMALL LETTER TLO +ABB1; C; 13E1; # CHEROKEE SMALL LETTER TLU +ABB2; C; 13E2; # CHEROKEE SMALL LETTER TLV +ABB3; C; 13E3; # CHEROKEE SMALL LETTER TSA +ABB4; C; 13E4; # CHEROKEE SMALL LETTER TSE +ABB5; C; 13E5; # CHEROKEE SMALL LETTER TSI +ABB6; C; 13E6; # CHEROKEE SMALL LETTER TSO +ABB7; C; 13E7; # CHEROKEE SMALL LETTER TSU +ABB8; C; 13E8; # CHEROKEE SMALL LETTER TSV +ABB9; C; 13E9; # CHEROKEE SMALL LETTER WA +ABBA; C; 13EA; # CHEROKEE SMALL LETTER WE +ABBB; C; 13EB; # CHEROKEE SMALL LETTER WI +ABBC; C; 13EC; # CHEROKEE SMALL LETTER WO +ABBD; C; 13ED; # CHEROKEE SMALL LETTER WU +ABBE; C; 13EE; # CHEROKEE SMALL LETTER WV +ABBF; C; 13EF; # CHEROKEE SMALL LETTER YA +FB00; F; 0066 0066; # LATIN SMALL LIGATURE FF +FB01; F; 0066 0069; # LATIN SMALL LIGATURE FI +FB02; F; 0066 006C; # LATIN SMALL LIGATURE FL +FB03; F; 0066 0066 0069; # LATIN SMALL LIGATURE FFI +FB04; F; 0066 0066 006C; # LATIN SMALL LIGATURE FFL +FB05; F; 0073 0074; # LATIN SMALL LIGATURE LONG S T +FB06; F; 0073 0074; # LATIN SMALL LIGATURE ST +FB13; F; 0574 0576; # ARMENIAN SMALL LIGATURE MEN NOW +FB14; F; 0574 0565; # ARMENIAN SMALL LIGATURE MEN ECH +FB15; F; 0574 056B; # ARMENIAN SMALL LIGATURE MEN INI +FB16; F; 057E 0576; # ARMENIAN SMALL LIGATURE VEW NOW +FB17; F; 0574 056D; # ARMENIAN SMALL LIGATURE MEN XEH +FF21; C; FF41; # FULLWIDTH LATIN CAPITAL LETTER A +FF22; C; FF42; # FULLWIDTH LATIN CAPITAL LETTER B +FF23; C; FF43; # FULLWIDTH LATIN CAPITAL LETTER C +FF24; C; FF44; # FULLWIDTH LATIN CAPITAL LETTER D +FF25; C; FF45; # FULLWIDTH LATIN CAPITAL LETTER E +FF26; C; FF46; # FULLWIDTH LATIN CAPITAL LETTER F +FF27; C; FF47; # FULLWIDTH LATIN CAPITAL LETTER G +FF28; C; FF48; # FULLWIDTH LATIN CAPITAL LETTER H +FF29; C; FF49; # FULLWIDTH LATIN CAPITAL LETTER I +FF2A; C; FF4A; # FULLWIDTH LATIN CAPITAL LETTER J +FF2B; C; FF4B; # FULLWIDTH LATIN CAPITAL LETTER K +FF2C; C; FF4C; # FULLWIDTH LATIN CAPITAL LETTER L +FF2D; C; FF4D; # FULLWIDTH LATIN CAPITAL LETTER M +FF2E; C; FF4E; # FULLWIDTH LATIN CAPITAL LETTER N +FF2F; C; FF4F; # FULLWIDTH LATIN CAPITAL LETTER O +FF30; C; FF50; # FULLWIDTH LATIN CAPITAL LETTER P +FF31; C; FF51; # FULLWIDTH LATIN CAPITAL LETTER Q +FF32; C; FF52; # FULLWIDTH LATIN CAPITAL LETTER R +FF33; C; FF53; # FULLWIDTH LATIN CAPITAL LETTER S +FF34; C; FF54; # FULLWIDTH LATIN CAPITAL LETTER T +FF35; C; FF55; # FULLWIDTH LATIN CAPITAL LETTER U +FF36; C; FF56; # FULLWIDTH LATIN CAPITAL LETTER V +FF37; C; FF57; # FULLWIDTH LATIN CAPITAL LETTER W +FF38; C; FF58; # FULLWIDTH LATIN CAPITAL LETTER X +FF39; C; FF59; # FULLWIDTH LATIN CAPITAL LETTER Y +FF3A; C; FF5A; # FULLWIDTH LATIN CAPITAL LETTER Z +10400; C; 10428; # DESERET CAPITAL LETTER LONG I +10401; C; 10429; # DESERET CAPITAL LETTER LONG E +10402; C; 1042A; # DESERET CAPITAL LETTER LONG A +10403; C; 1042B; # DESERET CAPITAL LETTER LONG AH +10404; C; 1042C; # DESERET CAPITAL LETTER LONG O +10405; C; 1042D; # DESERET CAPITAL LETTER LONG OO +10406; C; 1042E; # DESERET CAPITAL LETTER SHORT I +10407; C; 1042F; # DESERET CAPITAL LETTER SHORT E +10408; C; 10430; # DESERET CAPITAL LETTER SHORT A +10409; C; 10431; # DESERET CAPITAL LETTER SHORT AH +1040A; C; 10432; # DESERET CAPITAL LETTER SHORT O +1040B; C; 10433; # DESERET CAPITAL LETTER SHORT OO +1040C; C; 10434; # DESERET CAPITAL LETTER AY +1040D; C; 10435; # DESERET CAPITAL LETTER OW +1040E; C; 10436; # DESERET CAPITAL LETTER WU +1040F; C; 10437; # DESERET CAPITAL LETTER YEE +10410; C; 10438; # DESERET CAPITAL LETTER H +10411; C; 10439; # DESERET CAPITAL LETTER PEE +10412; C; 1043A; # DESERET CAPITAL LETTER BEE +10413; C; 1043B; # DESERET CAPITAL LETTER TEE +10414; C; 1043C; # DESERET CAPITAL LETTER DEE +10415; C; 1043D; # DESERET CAPITAL LETTER CHEE +10416; C; 1043E; # DESERET CAPITAL LETTER JEE +10417; C; 1043F; # DESERET CAPITAL LETTER KAY +10418; C; 10440; # DESERET CAPITAL LETTER GAY +10419; C; 10441; # DESERET CAPITAL LETTER EF +1041A; C; 10442; # DESERET CAPITAL LETTER VEE +1041B; C; 10443; # DESERET CAPITAL LETTER ETH +1041C; C; 10444; # DESERET CAPITAL LETTER THEE +1041D; C; 10445; # DESERET CAPITAL LETTER ES +1041E; C; 10446; # DESERET CAPITAL LETTER ZEE +1041F; C; 10447; # DESERET CAPITAL LETTER ESH +10420; C; 10448; # DESERET CAPITAL LETTER ZHEE +10421; C; 10449; # DESERET CAPITAL LETTER ER +10422; C; 1044A; # DESERET CAPITAL LETTER EL +10423; C; 1044B; # DESERET CAPITAL LETTER EM +10424; C; 1044C; # DESERET CAPITAL LETTER EN +10425; C; 1044D; # DESERET CAPITAL LETTER ENG +10426; C; 1044E; # DESERET CAPITAL LETTER OI +10427; C; 1044F; # DESERET CAPITAL LETTER EW +104B0; C; 104D8; # OSAGE CAPITAL LETTER A +104B1; C; 104D9; # OSAGE CAPITAL LETTER AI +104B2; C; 104DA; # OSAGE CAPITAL LETTER AIN +104B3; C; 104DB; # OSAGE CAPITAL LETTER AH +104B4; C; 104DC; # OSAGE CAPITAL LETTER BRA +104B5; C; 104DD; # OSAGE CAPITAL LETTER CHA +104B6; C; 104DE; # OSAGE CAPITAL LETTER EHCHA +104B7; C; 104DF; # OSAGE CAPITAL LETTER E +104B8; C; 104E0; # OSAGE CAPITAL LETTER EIN +104B9; C; 104E1; # OSAGE CAPITAL LETTER HA +104BA; C; 104E2; # OSAGE CAPITAL LETTER HYA +104BB; C; 104E3; # OSAGE CAPITAL LETTER I +104BC; C; 104E4; # OSAGE CAPITAL LETTER KA +104BD; C; 104E5; # OSAGE CAPITAL LETTER EHKA +104BE; C; 104E6; # OSAGE CAPITAL LETTER KYA +104BF; C; 104E7; # OSAGE CAPITAL LETTER LA +104C0; C; 104E8; # OSAGE CAPITAL LETTER MA +104C1; C; 104E9; # OSAGE CAPITAL LETTER NA +104C2; C; 104EA; # OSAGE CAPITAL LETTER O +104C3; C; 104EB; # OSAGE CAPITAL LETTER OIN +104C4; C; 104EC; # OSAGE CAPITAL LETTER PA +104C5; C; 104ED; # OSAGE CAPITAL LETTER EHPA +104C6; C; 104EE; # OSAGE CAPITAL LETTER SA +104C7; C; 104EF; # OSAGE CAPITAL LETTER SHA +104C8; C; 104F0; # OSAGE CAPITAL LETTER TA +104C9; C; 104F1; # OSAGE CAPITAL LETTER EHTA +104CA; C; 104F2; # OSAGE CAPITAL LETTER TSA +104CB; C; 104F3; # OSAGE CAPITAL LETTER EHTSA +104CC; C; 104F4; # OSAGE CAPITAL LETTER TSHA +104CD; C; 104F5; # OSAGE CAPITAL LETTER DHA +104CE; C; 104F6; # OSAGE CAPITAL LETTER U +104CF; C; 104F7; # OSAGE CAPITAL LETTER WA +104D0; C; 104F8; # OSAGE CAPITAL LETTER KHA +104D1; C; 104F9; # OSAGE CAPITAL LETTER GHA +104D2; C; 104FA; # OSAGE CAPITAL LETTER ZA +104D3; C; 104FB; # OSAGE CAPITAL LETTER ZHA +10C80; C; 10CC0; # OLD HUNGARIAN CAPITAL LETTER A +10C81; C; 10CC1; # OLD HUNGARIAN CAPITAL LETTER AA +10C82; C; 10CC2; # OLD HUNGARIAN CAPITAL LETTER EB +10C83; C; 10CC3; # OLD HUNGARIAN CAPITAL LETTER AMB +10C84; C; 10CC4; # OLD HUNGARIAN CAPITAL LETTER EC +10C85; C; 10CC5; # OLD HUNGARIAN CAPITAL LETTER ENC +10C86; C; 10CC6; # OLD HUNGARIAN CAPITAL LETTER ECS +10C87; C; 10CC7; # OLD HUNGARIAN CAPITAL LETTER ED +10C88; C; 10CC8; # OLD HUNGARIAN CAPITAL LETTER AND +10C89; C; 10CC9; # OLD HUNGARIAN CAPITAL LETTER E +10C8A; C; 10CCA; # OLD HUNGARIAN CAPITAL LETTER CLOSE E +10C8B; C; 10CCB; # OLD HUNGARIAN CAPITAL LETTER EE +10C8C; C; 10CCC; # OLD HUNGARIAN CAPITAL LETTER EF +10C8D; C; 10CCD; # OLD HUNGARIAN CAPITAL LETTER EG +10C8E; C; 10CCE; # OLD HUNGARIAN CAPITAL LETTER EGY +10C8F; C; 10CCF; # OLD HUNGARIAN CAPITAL LETTER EH +10C90; C; 10CD0; # OLD HUNGARIAN CAPITAL LETTER I +10C91; C; 10CD1; # OLD HUNGARIAN CAPITAL LETTER II +10C92; C; 10CD2; # OLD HUNGARIAN CAPITAL LETTER EJ +10C93; C; 10CD3; # OLD HUNGARIAN CAPITAL LETTER EK +10C94; C; 10CD4; # OLD HUNGARIAN CAPITAL LETTER AK +10C95; C; 10CD5; # OLD HUNGARIAN CAPITAL LETTER UNK +10C96; C; 10CD6; # OLD HUNGARIAN CAPITAL LETTER EL +10C97; C; 10CD7; # OLD HUNGARIAN CAPITAL LETTER ELY +10C98; C; 10CD8; # OLD HUNGARIAN CAPITAL LETTER EM +10C99; C; 10CD9; # OLD HUNGARIAN CAPITAL LETTER EN +10C9A; C; 10CDA; # OLD HUNGARIAN CAPITAL LETTER ENY +10C9B; C; 10CDB; # OLD HUNGARIAN CAPITAL LETTER O +10C9C; C; 10CDC; # OLD HUNGARIAN CAPITAL LETTER OO +10C9D; C; 10CDD; # OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG OE +10C9E; C; 10CDE; # OLD HUNGARIAN CAPITAL LETTER RUDIMENTA OE +10C9F; C; 10CDF; # OLD HUNGARIAN CAPITAL LETTER OEE +10CA0; C; 10CE0; # OLD HUNGARIAN CAPITAL LETTER EP +10CA1; C; 10CE1; # OLD HUNGARIAN CAPITAL LETTER EMP +10CA2; C; 10CE2; # OLD HUNGARIAN CAPITAL LETTER ER +10CA3; C; 10CE3; # OLD HUNGARIAN CAPITAL LETTER SHORT ER +10CA4; C; 10CE4; # OLD HUNGARIAN CAPITAL LETTER ES +10CA5; C; 10CE5; # OLD HUNGARIAN CAPITAL LETTER ESZ +10CA6; C; 10CE6; # OLD HUNGARIAN CAPITAL LETTER ET +10CA7; C; 10CE7; # OLD HUNGARIAN CAPITAL LETTER ENT +10CA8; C; 10CE8; # OLD HUNGARIAN CAPITAL LETTER ETY +10CA9; C; 10CE9; # OLD HUNGARIAN CAPITAL LETTER ECH +10CAA; C; 10CEA; # OLD HUNGARIAN CAPITAL LETTER U +10CAB; C; 10CEB; # OLD HUNGARIAN CAPITAL LETTER UU +10CAC; C; 10CEC; # OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG UE +10CAD; C; 10CED; # OLD HUNGARIAN CAPITAL LETTER RUDIMENTA UE +10CAE; C; 10CEE; # OLD HUNGARIAN CAPITAL LETTER EV +10CAF; C; 10CEF; # OLD HUNGARIAN CAPITAL LETTER EZ +10CB0; C; 10CF0; # OLD HUNGARIAN CAPITAL LETTER EZS +10CB1; C; 10CF1; # OLD HUNGARIAN CAPITAL LETTER ENT-SHAPED SIGN +10CB2; C; 10CF2; # OLD HUNGARIAN CAPITAL LETTER US +118A0; C; 118C0; # WARANG CITI CAPITAL LETTER NGAA +118A1; C; 118C1; # WARANG CITI CAPITAL LETTER A +118A2; C; 118C2; # WARANG CITI CAPITAL LETTER WI +118A3; C; 118C3; # WARANG CITI CAPITAL LETTER YU +118A4; C; 118C4; # WARANG CITI CAPITAL LETTER YA +118A5; C; 118C5; # WARANG CITI CAPITAL LETTER YO +118A6; C; 118C6; # WARANG CITI CAPITAL LETTER II +118A7; C; 118C7; # WARANG CITI CAPITAL LETTER UU +118A8; C; 118C8; # WARANG CITI CAPITAL LETTER E +118A9; C; 118C9; # WARANG CITI CAPITAL LETTER O +118AA; C; 118CA; # WARANG CITI CAPITAL LETTER ANG +118AB; C; 118CB; # WARANG CITI CAPITAL LETTER GA +118AC; C; 118CC; # WARANG CITI CAPITAL LETTER KO +118AD; C; 118CD; # WARANG CITI CAPITAL LETTER ENY +118AE; C; 118CE; # WARANG CITI CAPITAL LETTER YUJ +118AF; C; 118CF; # WARANG CITI CAPITAL LETTER UC +118B0; C; 118D0; # WARANG CITI CAPITAL LETTER ENN +118B1; C; 118D1; # WARANG CITI CAPITAL LETTER ODD +118B2; C; 118D2; # WARANG CITI CAPITAL LETTER TTE +118B3; C; 118D3; # WARANG CITI CAPITAL LETTER NUNG +118B4; C; 118D4; # WARANG CITI CAPITAL LETTER DA +118B5; C; 118D5; # WARANG CITI CAPITAL LETTER AT +118B6; C; 118D6; # WARANG CITI CAPITAL LETTER AM +118B7; C; 118D7; # WARANG CITI CAPITAL LETTER BU +118B8; C; 118D8; # WARANG CITI CAPITAL LETTER PU +118B9; C; 118D9; # WARANG CITI CAPITAL LETTER HIYO +118BA; C; 118DA; # WARANG CITI CAPITAL LETTER HOLO +118BB; C; 118DB; # WARANG CITI CAPITAL LETTER HORR +118BC; C; 118DC; # WARANG CITI CAPITAL LETTER HAR +118BD; C; 118DD; # WARANG CITI CAPITAL LETTER SSUU +118BE; C; 118DE; # WARANG CITI CAPITAL LETTER SII +118BF; C; 118DF; # WARANG CITI CAPITAL LETTER VIYO +1E900; C; 1E922; # ADLAM CAPITAL LETTER ALIF +1E901; C; 1E923; # ADLAM CAPITAL LETTER DAALI +1E902; C; 1E924; # ADLAM CAPITAL LETTER LAAM +1E903; C; 1E925; # ADLAM CAPITAL LETTER MIIM +1E904; C; 1E926; # ADLAM CAPITAL LETTER BA +1E905; C; 1E927; # ADLAM CAPITAL LETTER SINNYIIYHE +1E906; C; 1E928; # ADLAM CAPITAL LETTER PE +1E907; C; 1E929; # ADLAM CAPITAL LETTER BHE +1E908; C; 1E92A; # ADLAM CAPITAL LETTER RA +1E909; C; 1E92B; # ADLAM CAPITAL LETTER E +1E90A; C; 1E92C; # ADLAM CAPITAL LETTER FA +1E90B; C; 1E92D; # ADLAM CAPITAL LETTER I +1E90C; C; 1E92E; # ADLAM CAPITAL LETTER O +1E90D; C; 1E92F; # ADLAM CAPITAL LETTER DHA +1E90E; C; 1E930; # ADLAM CAPITAL LETTER YHE +1E90F; C; 1E931; # ADLAM CAPITAL LETTER WAW +1E910; C; 1E932; # ADLAM CAPITAL LETTER NUN +1E911; C; 1E933; # ADLAM CAPITAL LETTER KAF +1E912; C; 1E934; # ADLAM CAPITAL LETTER YA +1E913; C; 1E935; # ADLAM CAPITAL LETTER U +1E914; C; 1E936; # ADLAM CAPITAL LETTER JIIM +1E915; C; 1E937; # ADLAM CAPITAL LETTER CHI +1E916; C; 1E938; # ADLAM CAPITAL LETTER HA +1E917; C; 1E939; # ADLAM CAPITAL LETTER QAAF +1E918; C; 1E93A; # ADLAM CAPITAL LETTER GA +1E919; C; 1E93B; # ADLAM CAPITAL LETTER NYA +1E91A; C; 1E93C; # ADLAM CAPITAL LETTER TU +1E91B; C; 1E93D; # ADLAM CAPITAL LETTER NHA +1E91C; C; 1E93E; # ADLAM CAPITAL LETTER VA +1E91D; C; 1E93F; # ADLAM CAPITAL LETTER KHA +1E91E; C; 1E940; # ADLAM CAPITAL LETTER GBE +1E91F; C; 1E941; # ADLAM CAPITAL LETTER ZAL +1E920; C; 1E942; # ADLAM CAPITAL LETTER KPO +1E921; C; 1E943; # ADLAM CAPITAL LETTER SHA +# +# EOF diff --git a/rpython/rlib/unicodedata/CompositionExclusions-9.0.0.txt b/rpython/rlib/unicodedata/CompositionExclusions-9.0.0.txt new file mode 100644 --- /dev/null +++ b/rpython/rlib/unicodedata/CompositionExclusions-9.0.0.txt @@ -0,0 +1,208 @@ +# CompositionExclusions-9.0.0.txt +# Date: 2016-01-21, 22:00:00 GMT [KW, LI] +# © 2016 Unicode®, Inc. +# For terms of use, see http://www.unicode.org/terms_of_use.html +# +# Unicode Character Database +# For documentation, see http://www.unicode.org/reports/tr44/ +# +# This file lists the characters for the Composition Exclusion Table +# defined in UAX #15, Unicode Normalization Forms. +# +# This file is a normative contributory data file in the +# Unicode Character Database. +# +# For more information, see +# http://www.unicode.org/unicode/reports/tr15/#Primary_Exclusion_List_Table +# +# For a full derivation of composition exclusions, see the derived property +# Full_Composition_Exclusion in DerivedNormalizationProps.txt +# + +# ================================================ +# (1) Script Specifics +# +# This list of characters cannot be derived from the UnicodeData.txt file. +# ================================================ + +0958 # DEVANAGARI LETTER QA +0959 # DEVANAGARI LETTER KHHA +095A # DEVANAGARI LETTER GHHA +095B # DEVANAGARI LETTER ZA +095C # DEVANAGARI LETTER DDDHA +095D # DEVANAGARI LETTER RHA +095E # DEVANAGARI LETTER FA +095F # DEVANAGARI LETTER YYA +09DC # BENGALI LETTER RRA +09DD # BENGALI LETTER RHA +09DF # BENGALI LETTER YYA +0A33 # GURMUKHI LETTER LLA +0A36 # GURMUKHI LETTER SHA +0A59 # GURMUKHI LETTER KHHA +0A5A # GURMUKHI LETTER GHHA +0A5B # GURMUKHI LETTER ZA +0A5E # GURMUKHI LETTER FA +0B5C # ORIYA LETTER RRA +0B5D # ORIYA LETTER RHA +0F43 # TIBETAN LETTER GHA +0F4D # TIBETAN LETTER DDHA +0F52 # TIBETAN LETTER DHA +0F57 # TIBETAN LETTER BHA +0F5C # TIBETAN LETTER DZHA +0F69 # TIBETAN LETTER KSSA +0F76 # TIBETAN VOWEL SIGN VOCALIC R +0F78 # TIBETAN VOWEL SIGN VOCALIC L +0F93 # TIBETAN SUBJOINED LETTER GHA +0F9D # TIBETAN SUBJOINED LETTER DDHA +0FA2 # TIBETAN SUBJOINED LETTER DHA +0FA7 # TIBETAN SUBJOINED LETTER BHA +0FAC # TIBETAN SUBJOINED LETTER DZHA +0FB9 # TIBETAN SUBJOINED LETTER KSSA +FB1D # HEBREW LETTER YOD WITH HIRIQ +FB1F # HEBREW LIGATURE YIDDISH YOD YOD PATAH +FB2A # HEBREW LETTER SHIN WITH SHIN DOT +FB2B # HEBREW LETTER SHIN WITH SIN DOT +FB2C # HEBREW LETTER SHIN WITH DAGESH AND SHIN DOT +FB2D # HEBREW LETTER SHIN WITH DAGESH AND SIN DOT +FB2E # HEBREW LETTER ALEF WITH PATAH +FB2F # HEBREW LETTER ALEF WITH QAMATS +FB30 # HEBREW LETTER ALEF WITH MAPIQ +FB31 # HEBREW LETTER BET WITH DAGESH +FB32 # HEBREW LETTER GIMEL WITH DAGESH +FB33 # HEBREW LETTER DALET WITH DAGESH +FB34 # HEBREW LETTER HE WITH MAPIQ +FB35 # HEBREW LETTER VAV WITH DAGESH +FB36 # HEBREW LETTER ZAYIN WITH DAGESH +FB38 # HEBREW LETTER TET WITH DAGESH +FB39 # HEBREW LETTER YOD WITH DAGESH +FB3A # HEBREW LETTER FINAL KAF WITH DAGESH +FB3B # HEBREW LETTER KAF WITH DAGESH +FB3C # HEBREW LETTER LAMED WITH DAGESH +FB3E # HEBREW LETTER MEM WITH DAGESH +FB40 # HEBREW LETTER NUN WITH DAGESH +FB41 # HEBREW LETTER SAMEKH WITH DAGESH +FB43 # HEBREW LETTER FINAL PE WITH DAGESH +FB44 # HEBREW LETTER PE WITH DAGESH +FB46 # HEBREW LETTER TSADI WITH DAGESH +FB47 # HEBREW LETTER QOF WITH DAGESH +FB48 # HEBREW LETTER RESH WITH DAGESH +FB49 # HEBREW LETTER SHIN WITH DAGESH +FB4A # HEBREW LETTER TAV WITH DAGESH +FB4B # HEBREW LETTER VAV WITH HOLAM +FB4C # HEBREW LETTER BET WITH RAFE +FB4D # HEBREW LETTER KAF WITH RAFE +FB4E # HEBREW LETTER PE WITH RAFE + +# Total code points: 67 + +# ================================================ +# (2) Post Composition Version precomposed characters +# +# These characters cannot be derived solely from the UnicodeData.txt file +# in this version of Unicode. +# +# Note that characters added to the standard after the +# Composition Version and which have canonical decomposition mappings +# are not automatically added to this list of Post Composition +# Version precomposed characters. +# ================================================ + +2ADC # FORKING +1D15E # MUSICAL SYMBOL HALF NOTE +1D15F # MUSICAL SYMBOL QUARTER NOTE +1D160 # MUSICAL SYMBOL EIGHTH NOTE +1D161 # MUSICAL SYMBOL SIXTEENTH NOTE +1D162 # MUSICAL SYMBOL THIRTY-SECOND NOTE +1D163 # MUSICAL SYMBOL SIXTY-FOURTH NOTE +1D164 # MUSICAL SYMBOL ONE HUNDRED TWENTY-EIGHTH NOTE +1D1BB # MUSICAL SYMBOL MINIMA +1D1BC # MUSICAL SYMBOL MINIMA BLACK +1D1BD # MUSICAL SYMBOL SEMIMINIMA WHITE +1D1BE # MUSICAL SYMBOL SEMIMINIMA BLACK +1D1BF # MUSICAL SYMBOL FUSA WHITE +1D1C0 # MUSICAL SYMBOL FUSA BLACK + +# Total code points: 14 + +# ================================================ +# (3) Singleton Decompositions +# +# These characters can be derived from the UnicodeData.txt file +# by including all canonically decomposable characters whose +# canonical decomposition consists of a single character. +# +# These characters are simply quoted here for reference. +# See also Full_Composition_Exclusion in DerivedNormalizationProps.txt +# ================================================ + +# 0340..0341 [2] COMBINING GRAVE TONE MARK..COMBINING ACUTE TONE MARK +# 0343 COMBINING GREEK KORONIS +# 0374 GREEK NUMERAL SIGN +# 037E GREEK QUESTION MARK +# 0387 GREEK ANO TELEIA +# 1F71 GREEK SMALL LETTER ALPHA WITH OXIA +# 1F73 GREEK SMALL LETTER EPSILON WITH OXIA +# 1F75 GREEK SMALL LETTER ETA WITH OXIA +# 1F77 GREEK SMALL LETTER IOTA WITH OXIA +# 1F79 GREEK SMALL LETTER OMICRON WITH OXIA +# 1F7B GREEK SMALL LETTER UPSILON WITH OXIA +# 1F7D GREEK SMALL LETTER OMEGA WITH OXIA +# 1FBB GREEK CAPITAL LETTER ALPHA WITH OXIA +# 1FBE GREEK PROSGEGRAMMENI +# 1FC9 GREEK CAPITAL LETTER EPSILON WITH OXIA +# 1FCB GREEK CAPITAL LETTER ETA WITH OXIA +# 1FD3 GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +# 1FDB GREEK CAPITAL LETTER IOTA WITH OXIA +# 1FE3 GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA +# 1FEB GREEK CAPITAL LETTER UPSILON WITH OXIA +# 1FEE..1FEF [2] GREEK DIALYTIKA AND OXIA..GREEK VARIA +# 1FF9 GREEK CAPITAL LETTER OMICRON WITH OXIA +# 1FFB GREEK CAPITAL LETTER OMEGA WITH OXIA +# 1FFD GREEK OXIA +# 2000..2001 [2] EN QUAD..EM QUAD +# 2126 OHM SIGN +# 212A..212B [2] KELVIN SIGN..ANGSTROM SIGN +# 2329 LEFT-POINTING ANGLE BRACKET +# 232A RIGHT-POINTING ANGLE BRACKET +# F900..FA0D [270] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA0D +# FA10 CJK COMPATIBILITY IDEOGRAPH-FA10 +# FA12 CJK COMPATIBILITY IDEOGRAPH-FA12 +# FA15..FA1E [10] CJK COMPATIBILITY IDEOGRAPH-FA15..CJK COMPATIBILITY IDEOGRAPH-FA1E +# FA20 CJK COMPATIBILITY IDEOGRAPH-FA20 +# FA22 CJK COMPATIBILITY IDEOGRAPH-FA22 +# FA25..FA26 [2] CJK COMPATIBILITY IDEOGRAPH-FA25..CJK COMPATIBILITY IDEOGRAPH-FA26 +# FA2A..FA6D [68] CJK COMPATIBILITY IDEOGRAPH-FA2A..CJK COMPATIBILITY IDEOGRAPH-FA6D +# FA70..FAD9 [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COMPATIBILITY IDEOGRAPH-FAD9 +# 2F800..2FA1D [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D + +# Total code points: 1035 + +# ================================================ +# (4) Non-Starter Decompositions +# +# These characters can be derived from the UnicodeData.txt file +# by including each expanding canonical decomposition +# (i.e., those which canonically decompose to a sequence +# of characters instead of a single character), such that: +# +# A. The character is not a Starter. +# +# OR (inclusive) +# +# B. The character's canonical decomposition begins +# with a character that is not a Starter. +# +# Note that a "Starter" is any character with a zero combining class. +# +# These characters are simply quoted here for reference. +# See also Full_Composition_Exclusion in DerivedNormalizationProps.txt +# ================================================ + +# 0344 COMBINING GREEK DIALYTIKA TONOS +# 0F73 TIBETAN VOWEL SIGN II +# 0F75 TIBETAN VOWEL SIGN UU +# 0F81 TIBETAN VOWEL SIGN REVERSED II + +# Total code points: 4 + +# EOF diff --git a/rpython/rlib/unicodedata/DerivedCoreProperties-9.0.0.txt b/rpython/rlib/unicodedata/DerivedCoreProperties-9.0.0.txt new file mode 100644 --- /dev/null +++ b/rpython/rlib/unicodedata/DerivedCoreProperties-9.0.0.txt @@ -0,0 +1,11309 @@ +# DerivedCoreProperties-9.0.0.txt +# Date: 2016-06-01, 10:34:24 GMT +# © 2016 Unicode®, Inc. +# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. +# For terms of use, see http://www.unicode.org/terms_of_use.html +# +# Unicode Character Database +# For documentation, see http://www.unicode.org/reports/tr44/ + +# ================================================ + +# Derived Property: Math +# Generated from: Sm + Other_Math + +002B ; Math # Sm PLUS SIGN +003C..003E ; Math # Sm [3] LESS-THAN SIGN..GREATER-THAN SIGN +005E ; Math # Sk CIRCUMFLEX ACCENT +007C ; Math # Sm VERTICAL LINE +007E ; Math # Sm TILDE +00AC ; Math # Sm NOT SIGN +00B1 ; Math # Sm PLUS-MINUS SIGN +00D7 ; Math # Sm MULTIPLICATION SIGN +00F7 ; Math # Sm DIVISION SIGN +03D0..03D2 ; Math # L& [3] GREEK BETA SYMBOL..GREEK UPSILON WITH HOOK SYMBOL +03D5 ; Math # L& GREEK PHI SYMBOL +03F0..03F1 ; Math # L& [2] GREEK KAPPA SYMBOL..GREEK RHO SYMBOL +03F4..03F5 ; Math # L& [2] GREEK CAPITAL THETA SYMBOL..GREEK LUNATE EPSILON SYMBOL +03F6 ; Math # Sm GREEK REVERSED LUNATE EPSILON SYMBOL +0606..0608 ; Math # Sm [3] ARABIC-INDIC CUBE ROOT..ARABIC RAY +2016 ; Math # Po DOUBLE VERTICAL LINE +2032..2034 ; Math # Po [3] PRIME..TRIPLE PRIME +2040 ; Math # Pc CHARACTER TIE +2044 ; Math # Sm FRACTION SLASH +2052 ; Math # Sm COMMERCIAL MINUS SIGN +2061..2064 ; Math # Cf [4] FUNCTION APPLICATION..INVISIBLE PLUS +207A..207C ; Math # Sm [3] SUPERSCRIPT PLUS SIGN..SUPERSCRIPT EQUALS SIGN +207D ; Math # Ps SUPERSCRIPT LEFT PARENTHESIS +207E ; Math # Pe SUPERSCRIPT RIGHT PARENTHESIS +208A..208C ; Math # Sm [3] SUBSCRIPT PLUS SIGN..SUBSCRIPT EQUALS SIGN +208D ; Math # Ps SUBSCRIPT LEFT PARENTHESIS +208E ; Math # Pe SUBSCRIPT RIGHT PARENTHESIS +20D0..20DC ; Math # Mn [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE +20E1 ; Math # Mn COMBINING LEFT RIGHT ARROW ABOVE +20E5..20E6 ; Math # Mn [2] COMBINING REVERSE SOLIDUS OVERLAY..COMBINING DOUBLE VERTICAL STROKE OVERLAY +20EB..20EF ; Math # Mn [5] COMBINING LONG DOUBLE SOLIDUS OVERLAY..COMBINING RIGHT ARROW BELOW +2102 ; Math # L& DOUBLE-STRUCK CAPITAL C +2107 ; Math # L& EULER CONSTANT +210A..2113 ; Math # L& [10] SCRIPT SMALL G..SCRIPT SMALL L +2115 ; Math # L& DOUBLE-STRUCK CAPITAL N +2118 ; Math # Sm SCRIPT CAPITAL P +2119..211D ; Math # L& [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R +2124 ; Math # L& DOUBLE-STRUCK CAPITAL Z +2128 ; Math # L& BLACK-LETTER CAPITAL Z +2129 ; Math # So TURNED GREEK SMALL LETTER IOTA +212C..212D ; Math # L& [2] SCRIPT CAPITAL B..BLACK-LETTER CAPITAL C +212F..2131 ; Math # L& [3] SCRIPT SMALL E..SCRIPT CAPITAL F +2133..2134 ; Math # L& [2] SCRIPT CAPITAL M..SCRIPT SMALL O +2135..2138 ; Math # Lo [4] ALEF SYMBOL..DALET SYMBOL +213C..213F ; Math # L& [4] DOUBLE-STRUCK SMALL PI..DOUBLE-STRUCK CAPITAL PI +2140..2144 ; Math # Sm [5] DOUBLE-STRUCK N-ARY SUMMATION..TURNED SANS-SERIF CAPITAL Y +2145..2149 ; Math # L& [5] DOUBLE-STRUCK ITALIC CAPITAL D..DOUBLE-STRUCK ITALIC SMALL J +214B ; Math # Sm TURNED AMPERSAND +2190..2194 ; Math # Sm [5] LEFTWARDS ARROW..LEFT RIGHT ARROW +2195..2199 ; Math # So [5] UP DOWN ARROW..SOUTH WEST ARROW +219A..219B ; Math # Sm [2] LEFTWARDS ARROW WITH STROKE..RIGHTWARDS ARROW WITH STROKE +219C..219F ; Math # So [4] LEFTWARDS WAVE ARROW..UPWARDS TWO HEADED ARROW +21A0 ; Math # Sm RIGHTWARDS TWO HEADED ARROW +21A1..21A2 ; Math # So [2] DOWNWARDS TWO HEADED ARROW..LEFTWARDS ARROW WITH TAIL +21A3 ; Math # Sm RIGHTWARDS ARROW WITH TAIL +21A4..21A5 ; Math # So [2] LEFTWARDS ARROW FROM BAR..UPWARDS ARROW FROM BAR +21A6 ; Math # Sm RIGHTWARDS ARROW FROM BAR +21A7 ; Math # So DOWNWARDS ARROW FROM BAR +21A9..21AD ; Math # So [5] LEFTWARDS ARROW WITH HOOK..LEFT RIGHT WAVE ARROW +21AE ; Math # Sm LEFT RIGHT ARROW WITH STROKE +21B0..21B1 ; Math # So [2] UPWARDS ARROW WITH TIP LEFTWARDS..UPWARDS ARROW WITH TIP RIGHTWARDS +21B6..21B7 ; Math # So [2] ANTICLOCKWISE TOP SEMICIRCLE ARROW..CLOCKWISE TOP SEMICIRCLE ARROW +21BC..21CD ; Math # So [18] LEFTWARDS HARPOON WITH BARB UPWARDS..LEFTWARDS DOUBLE ARROW WITH STROKE +21CE..21CF ; Math # Sm [2] LEFT RIGHT DOUBLE ARROW WITH STROKE..RIGHTWARDS DOUBLE ARROW WITH STROKE +21D0..21D1 ; Math # So [2] LEFTWARDS DOUBLE ARROW..UPWARDS DOUBLE ARROW +21D2 ; Math # Sm RIGHTWARDS DOUBLE ARROW +21D3 ; Math # So DOWNWARDS DOUBLE ARROW +21D4 ; Math # Sm LEFT RIGHT DOUBLE ARROW +21D5..21DB ; Math # So [7] UP DOWN DOUBLE ARROW..RIGHTWARDS TRIPLE ARROW +21DD ; Math # So RIGHTWARDS SQUIGGLE ARROW +21E4..21E5 ; Math # So [2] LEFTWARDS ARROW TO BAR..RIGHTWARDS ARROW TO BAR +21F4..22FF ; Math # Sm [268] RIGHT ARROW WITH SMALL CIRCLE..Z NOTATION BAG MEMBERSHIP +2308 ; Math # Ps LEFT CEILING +2309 ; Math # Pe RIGHT CEILING +230A ; Math # Ps LEFT FLOOR +230B ; Math # Pe RIGHT FLOOR +2320..2321 ; Math # Sm [2] TOP HALF INTEGRAL..BOTTOM HALF INTEGRAL +237C ; Math # Sm RIGHT ANGLE WITH DOWNWARDS ZIGZAG ARROW +239B..23B3 ; Math # Sm [25] LEFT PARENTHESIS UPPER HOOK..SUMMATION BOTTOM +23B4..23B5 ; Math # So [2] TOP SQUARE BRACKET..BOTTOM SQUARE BRACKET +23B7 ; Math # So RADICAL SYMBOL BOTTOM +23D0 ; Math # So VERTICAL LINE EXTENSION +23DC..23E1 ; Math # Sm [6] TOP PARENTHESIS..BOTTOM TORTOISE SHELL BRACKET +23E2 ; Math # So WHITE TRAPEZIUM +25A0..25A1 ; Math # So [2] BLACK SQUARE..WHITE SQUARE +25AE..25B6 ; Math # So [9] BLACK VERTICAL RECTANGLE..BLACK RIGHT-POINTING TRIANGLE +25B7 ; Math # Sm WHITE RIGHT-POINTING TRIANGLE +25BC..25C0 ; Math # So [5] BLACK DOWN-POINTING TRIANGLE..BLACK LEFT-POINTING TRIANGLE +25C1 ; Math # Sm WHITE LEFT-POINTING TRIANGLE +25C6..25C7 ; Math # So [2] BLACK DIAMOND..WHITE DIAMOND +25CA..25CB ; Math # So [2] LOZENGE..WHITE CIRCLE +25CF..25D3 ; Math # So [5] BLACK CIRCLE..CIRCLE WITH UPPER HALF BLACK +25E2 ; Math # So BLACK LOWER RIGHT TRIANGLE +25E4 ; Math # So BLACK UPPER LEFT TRIANGLE +25E7..25EC ; Math # So [6] SQUARE WITH LEFT HALF BLACK..WHITE UP-POINTING TRIANGLE WITH DOT +25F8..25FF ; Math # Sm [8] UPPER LEFT TRIANGLE..LOWER RIGHT TRIANGLE +2605..2606 ; Math # So [2] BLACK STAR..WHITE STAR +2640 ; Math # So FEMALE SIGN +2642 ; Math # So MALE SIGN +2660..2663 ; Math # So [4] BLACK SPADE SUIT..BLACK CLUB SUIT +266D..266E ; Math # So [2] MUSIC FLAT SIGN..MUSIC NATURAL SIGN +266F ; Math # Sm MUSIC SHARP SIGN +27C0..27C4 ; Math # Sm [5] THREE DIMENSIONAL ANGLE..OPEN SUPERSET +27C5 ; Math # Ps LEFT S-SHAPED BAG DELIMITER +27C6 ; Math # Pe RIGHT S-SHAPED BAG DELIMITER +27C7..27E5 ; Math # Sm [31] OR WITH DOT INSIDE..WHITE SQUARE WITH RIGHTWARDS TICK +27E6 ; Math # Ps MATHEMATICAL LEFT WHITE SQUARE BRACKET +27E7 ; Math # Pe MATHEMATICAL RIGHT WHITE SQUARE BRACKET +27E8 ; Math # Ps MATHEMATICAL LEFT ANGLE BRACKET +27E9 ; Math # Pe MATHEMATICAL RIGHT ANGLE BRACKET +27EA ; Math # Ps MATHEMATICAL LEFT DOUBLE ANGLE BRACKET +27EB ; Math # Pe MATHEMATICAL RIGHT DOUBLE ANGLE BRACKET +27EC ; Math # Ps MATHEMATICAL LEFT WHITE TORTOISE SHELL BRACKET +27ED ; Math # Pe MATHEMATICAL RIGHT WHITE TORTOISE SHELL BRACKET +27EE ; Math # Ps MATHEMATICAL LEFT FLATTENED PARENTHESIS +27EF ; Math # Pe MATHEMATICAL RIGHT FLATTENED PARENTHESIS +27F0..27FF ; Math # Sm [16] UPWARDS QUADRUPLE ARROW..LONG RIGHTWARDS SQUIGGLE ARROW +2900..2982 ; Math # Sm [131] RIGHTWARDS TWO-HEADED ARROW WITH VERTICAL STROKE..Z NOTATION TYPE COLON +2983 ; Math # Ps LEFT WHITE CURLY BRACKET +2984 ; Math # Pe RIGHT WHITE CURLY BRACKET +2985 ; Math # Ps LEFT WHITE PARENTHESIS +2986 ; Math # Pe RIGHT WHITE PARENTHESIS +2987 ; Math # Ps Z NOTATION LEFT IMAGE BRACKET +2988 ; Math # Pe Z NOTATION RIGHT IMAGE BRACKET +2989 ; Math # Ps Z NOTATION LEFT BINDING BRACKET +298A ; Math # Pe Z NOTATION RIGHT BINDING BRACKET +298B ; Math # Ps LEFT SQUARE BRACKET WITH UNDERBAR +298C ; Math # Pe RIGHT SQUARE BRACKET WITH UNDERBAR +298D ; Math # Ps LEFT SQUARE BRACKET WITH TICK IN TOP CORNER +298E ; Math # Pe RIGHT SQUARE BRACKET WITH TICK IN BOTTOM CORNER +298F ; Math # Ps LEFT SQUARE BRACKET WITH TICK IN BOTTOM CORNER +2990 ; Math # Pe RIGHT SQUARE BRACKET WITH TICK IN TOP CORNER +2991 ; Math # Ps LEFT ANGLE BRACKET WITH DOT +2992 ; Math # Pe RIGHT ANGLE BRACKET WITH DOT +2993 ; Math # Ps LEFT ARC LESS-THAN BRACKET +2994 ; Math # Pe RIGHT ARC GREATER-THAN BRACKET +2995 ; Math # Ps DOUBLE LEFT ARC GREATER-THAN BRACKET +2996 ; Math # Pe DOUBLE RIGHT ARC LESS-THAN BRACKET +2997 ; Math # Ps LEFT BLACK TORTOISE SHELL BRACKET +2998 ; Math # Pe RIGHT BLACK TORTOISE SHELL BRACKET +2999..29D7 ; Math # Sm [63] DOTTED FENCE..BLACK HOURGLASS +29D8 ; Math # Ps LEFT WIGGLY FENCE +29D9 ; Math # Pe RIGHT WIGGLY FENCE +29DA ; Math # Ps LEFT DOUBLE WIGGLY FENCE +29DB ; Math # Pe RIGHT DOUBLE WIGGLY FENCE +29DC..29FB ; Math # Sm [32] INCOMPLETE INFINITY..TRIPLE PLUS +29FC ; Math # Ps LEFT-POINTING CURVED ANGLE BRACKET +29FD ; Math # Pe RIGHT-POINTING CURVED ANGLE BRACKET +29FE..2AFF ; Math # Sm [258] TINY..N-ARY WHITE VERTICAL BAR +2B30..2B44 ; Math # Sm [21] LEFT ARROW WITH SMALL CIRCLE..RIGHTWARDS ARROW THROUGH SUPERSET +2B47..2B4C ; Math # Sm [6] REVERSE TILDE OPERATOR ABOVE RIGHTWARDS ARROW..RIGHTWARDS ARROW ABOVE REVERSE TILDE OPERATOR +FB29 ; Math # Sm HEBREW LETTER ALTERNATIVE PLUS SIGN +FE61 ; Math # Po SMALL ASTERISK +FE62 ; Math # Sm SMALL PLUS SIGN +FE63 ; Math # Pd SMALL HYPHEN-MINUS +FE64..FE66 ; Math # Sm [3] SMALL LESS-THAN SIGN..SMALL EQUALS SIGN +FE68 ; Math # Po SMALL REVERSE SOLIDUS +FF0B ; Math # Sm FULLWIDTH PLUS SIGN +FF1C..FF1E ; Math # Sm [3] FULLWIDTH LESS-THAN SIGN..FULLWIDTH GREATER-THAN SIGN +FF3C ; Math # Po FULLWIDTH REVERSE SOLIDUS +FF3E ; Math # Sk FULLWIDTH CIRCUMFLEX ACCENT +FF5C ; Math # Sm FULLWIDTH VERTICAL LINE +FF5E ; Math # Sm FULLWIDTH TILDE +FFE2 ; Math # Sm FULLWIDTH NOT SIGN +FFE9..FFEC ; Math # Sm [4] HALFWIDTH LEFTWARDS ARROW..HALFWIDTH DOWNWARDS ARROW +1D400..1D454 ; Math # L& [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G +1D456..1D49C ; Math # L& [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A +1D49E..1D49F ; Math # L& [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D +1D4A2 ; Math # L& MATHEMATICAL SCRIPT CAPITAL G +1D4A5..1D4A6 ; Math # L& [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K +1D4A9..1D4AC ; Math # L& [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q +1D4AE..1D4B9 ; Math # L& [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D +1D4BB ; Math # L& MATHEMATICAL SCRIPT SMALL F +1D4BD..1D4C3 ; Math # L& [7] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL N +1D4C5..1D505 ; Math # L& [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B +1D507..1D50A ; Math # L& [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G +1D50D..1D514 ; Math # L& [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q +1D516..1D51C ; Math # L& [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y +1D51E..1D539 ; Math # L& [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B +1D53B..1D53E ; Math # L& [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G +1D540..1D544 ; Math # L& [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M +1D546 ; Math # L& MATHEMATICAL DOUBLE-STRUCK CAPITAL O +1D54A..1D550 ; Math # L& [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y +1D552..1D6A5 ; Math # L& [340] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL ITALIC SMALL DOTLESS J +1D6A8..1D6C0 ; Math # L& [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA +1D6C1 ; Math # Sm MATHEMATICAL BOLD NABLA +1D6C2..1D6DA ; Math # L& [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA +1D6DB ; Math # Sm MATHEMATICAL BOLD PARTIAL DIFFERENTIAL +1D6DC..1D6FA ; Math # L& [31] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL ITALIC CAPITAL OMEGA +1D6FB ; Math # Sm MATHEMATICAL ITALIC NABLA +1D6FC..1D714 ; Math # L& [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA +1D715 ; Math # Sm MATHEMATICAL ITALIC PARTIAL DIFFERENTIAL +1D716..1D734 ; Math # L& [31] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA +1D735 ; Math # Sm MATHEMATICAL BOLD ITALIC NABLA +1D736..1D74E ; Math # L& [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA +1D74F ; Math # Sm MATHEMATICAL BOLD ITALIC PARTIAL DIFFERENTIAL +1D750..1D76E ; Math # L& [31] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA +1D76F ; Math # Sm MATHEMATICAL SANS-SERIF BOLD NABLA +1D770..1D788 ; Math # L& [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA +1D789 ; Math # Sm MATHEMATICAL SANS-SERIF BOLD PARTIAL DIFFERENTIAL +1D78A..1D7A8 ; Math # L& [31] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA +1D7A9 ; Math # Sm MATHEMATICAL SANS-SERIF BOLD ITALIC NABLA +1D7AA..1D7C2 ; Math # L& [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA +1D7C3 ; Math # Sm MATHEMATICAL SANS-SERIF BOLD ITALIC PARTIAL DIFFERENTIAL +1D7C4..1D7CB ; Math # L& [8] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD SMALL DIGAMMA +1D7CE..1D7FF ; Math # Nd [50] MATHEMATICAL BOLD DIGIT ZERO..MATHEMATICAL MONOSPACE DIGIT NINE +1EE00..1EE03 ; Math # Lo [4] ARABIC MATHEMATICAL ALEF..ARABIC MATHEMATICAL DAL +1EE05..1EE1F ; Math # Lo [27] ARABIC MATHEMATICAL WAW..ARABIC MATHEMATICAL DOTLESS QAF +1EE21..1EE22 ; Math # Lo [2] ARABIC MATHEMATICAL INITIAL BEH..ARABIC MATHEMATICAL INITIAL JEEM +1EE24 ; Math # Lo ARABIC MATHEMATICAL INITIAL HEH +1EE27 ; Math # Lo ARABIC MATHEMATICAL INITIAL HAH +1EE29..1EE32 ; Math # Lo [10] ARABIC MATHEMATICAL INITIAL YEH..ARABIC MATHEMATICAL INITIAL QAF +1EE34..1EE37 ; Math # Lo [4] ARABIC MATHEMATICAL INITIAL SHEEN..ARABIC MATHEMATICAL INITIAL KHAH +1EE39 ; Math # Lo ARABIC MATHEMATICAL INITIAL DAD +1EE3B ; Math # Lo ARABIC MATHEMATICAL INITIAL GHAIN +1EE42 ; Math # Lo ARABIC MATHEMATICAL TAILED JEEM +1EE47 ; Math # Lo ARABIC MATHEMATICAL TAILED HAH +1EE49 ; Math # Lo ARABIC MATHEMATICAL TAILED YEH +1EE4B ; Math # Lo ARABIC MATHEMATICAL TAILED LAM +1EE4D..1EE4F ; Math # Lo [3] ARABIC MATHEMATICAL TAILED NOON..ARABIC MATHEMATICAL TAILED AIN +1EE51..1EE52 ; Math # Lo [2] ARABIC MATHEMATICAL TAILED SAD..ARABIC MATHEMATICAL TAILED QAF +1EE54 ; Math # Lo ARABIC MATHEMATICAL TAILED SHEEN +1EE57 ; Math # Lo ARABIC MATHEMATICAL TAILED KHAH +1EE59 ; Math # Lo ARABIC MATHEMATICAL TAILED DAD +1EE5B ; Math # Lo ARABIC MATHEMATICAL TAILED GHAIN +1EE5D ; Math # Lo ARABIC MATHEMATICAL TAILED DOTLESS NOON +1EE5F ; Math # Lo ARABIC MATHEMATICAL TAILED DOTLESS QAF +1EE61..1EE62 ; Math # Lo [2] ARABIC MATHEMATICAL STRETCHED BEH..ARABIC MATHEMATICAL STRETCHED JEEM +1EE64 ; Math # Lo ARABIC MATHEMATICAL STRETCHED HEH +1EE67..1EE6A ; Math # Lo [4] ARABIC MATHEMATICAL STRETCHED HAH..ARABIC MATHEMATICAL STRETCHED KAF +1EE6C..1EE72 ; Math # Lo [7] ARABIC MATHEMATICAL STRETCHED MEEM..ARABIC MATHEMATICAL STRETCHED QAF +1EE74..1EE77 ; Math # Lo [4] ARABIC MATHEMATICAL STRETCHED SHEEN..ARABIC MATHEMATICAL STRETCHED KHAH +1EE79..1EE7C ; Math # Lo [4] ARABIC MATHEMATICAL STRETCHED DAD..ARABIC MATHEMATICAL STRETCHED DOTLESS BEH +1EE7E ; Math # Lo ARABIC MATHEMATICAL STRETCHED DOTLESS FEH +1EE80..1EE89 ; Math # Lo [10] ARABIC MATHEMATICAL LOOPED ALEF..ARABIC MATHEMATICAL LOOPED YEH +1EE8B..1EE9B ; Math # Lo [17] ARABIC MATHEMATICAL LOOPED LAM..ARABIC MATHEMATICAL LOOPED GHAIN +1EEA1..1EEA3 ; Math # Lo [3] ARABIC MATHEMATICAL DOUBLE-STRUCK BEH..ARABIC MATHEMATICAL DOUBLE-STRUCK DAL +1EEA5..1EEA9 ; Math # Lo [5] ARABIC MATHEMATICAL DOUBLE-STRUCK WAW..ARABIC MATHEMATICAL DOUBLE-STRUCK YEH +1EEAB..1EEBB ; Math # Lo [17] ARABIC MATHEMATICAL DOUBLE-STRUCK LAM..ARABIC MATHEMATICAL DOUBLE-STRUCK GHAIN +1EEF0..1EEF1 ; Math # Sm [2] ARABIC MATHEMATICAL OPERATOR MEEM WITH HAH WITH TATWEEL..ARABIC MATHEMATICAL OPERATOR HAH WITH DAL + +# Total code points: 2310 + +# ================================================ + +# Derived Property: Alphabetic +# Generated from: Uppercase + Lowercase + Lt + Lm + Lo + Nl + Other_Alphabetic + +0041..005A ; Alphabetic # L& [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z +0061..007A ; Alphabetic # L& [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z +00AA ; Alphabetic # Lo FEMININE ORDINAL INDICATOR +00B5 ; Alphabetic # L& MICRO SIGN +00BA ; Alphabetic # Lo MASCULINE ORDINAL INDICATOR +00C0..00D6 ; Alphabetic # L& [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS +00D8..00F6 ; Alphabetic # L& [31] LATIN CAPITAL LETTER O WITH STROKE..LATIN SMALL LETTER O WITH DIAERESIS +00F8..01BA ; Alphabetic # L& [195] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER EZH WITH TAIL +01BB ; Alphabetic # Lo LATIN LETTER TWO WITH STROKE +01BC..01BF ; Alphabetic # L& [4] LATIN CAPITAL LETTER TONE FIVE..LATIN LETTER WYNN +01C0..01C3 ; Alphabetic # Lo [4] LATIN LETTER DENTAL CLICK..LATIN LETTER RETROFLEX CLICK +01C4..0293 ; Alphabetic # L& [208] LATIN CAPITAL LETTER DZ WITH CARON..LATIN SMALL LETTER EZH WITH CURL +0294 ; Alphabetic # Lo LATIN LETTER GLOTTAL STOP +0295..02AF ; Alphabetic # L& [27] LATIN LETTER PHARYNGEAL VOICED FRICATIVE..LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL +02B0..02C1 ; Alphabetic # Lm [18] MODIFIER LETTER SMALL H..MODIFIER LETTER REVERSED GLOTTAL STOP +02C6..02D1 ; Alphabetic # Lm [12] MODIFIER LETTER CIRCUMFLEX ACCENT..MODIFIER LETTER HALF TRIANGULAR COLON +02E0..02E4 ; Alphabetic # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +02EC ; Alphabetic # Lm MODIFIER LETTER VOICING +02EE ; Alphabetic # Lm MODIFIER LETTER DOUBLE APOSTROPHE +0345 ; Alphabetic # Mn COMBINING GREEK YPOGEGRAMMENI From pypy.commits at gmail.com Sat Jan 6 12:15:22 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 06 Jan 2018 09:15:22 -0800 (PST) Subject: [pypy-commit] extradoc extradoc: add topics for sprint (from IRC and more) Message-ID: <5a51042a.85631c0a.3f69b.2385@mx.google.com> Author: Matti Picus Branch: extradoc Changeset: r5857:b851faa11400 Date: 2018-01-06 19:15 +0200 http://bitbucket.org/pypy/extradoc/changeset/b851faa11400/ Log: add topics for sprint (from IRC and more) diff --git a/planning/sprint-leysin-2018-notes.rst b/planning/sprint-leysin-2018-notes.rst --- a/planning/sprint-leysin-2018-notes.rst +++ b/planning/sprint-leysin-2018-notes.rst @@ -6,3 +6,8 @@ - make win32 builds green - make packaging more like cpython/portable builds - get CI builders for PyPy into mainstream projects (Numpy, Scipy, lxml, uwsgi) +- get more of scientific stack working (tensorflow?) +- cpyext performance improvements +- General 3.5 and 3.6 improvements +- JIT topics: guard-compatible, and the subsequent research project to save and reuse traces across processes +- finish unicode-uint8 From pypy.commits at gmail.com Sat Jan 6 12:38:03 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 06 Jan 2018 09:38:03 -0800 (PST) Subject: [pypy-commit] extradoc extradoc: fix and add one more Message-ID: <5a51097b.131f1c0a.4d123.fb76@mx.google.com> Author: Matti Picus Branch: extradoc Changeset: r5858:a9aa68e8a62c Date: 2018-01-06 19:27 +0200 http://bitbucket.org/pypy/extradoc/changeset/a9aa68e8a62c/ Log: fix and add one more diff --git a/planning/sprint-leysin-2018-notes.rst b/planning/sprint-leysin-2018-notes.rst --- a/planning/sprint-leysin-2018-notes.rst +++ b/planning/sprint-leysin-2018-notes.rst @@ -10,4 +10,5 @@ - cpyext performance improvements - General 3.5 and 3.6 improvements - JIT topics: guard-compatible, and the subsequent research project to save and reuse traces across processes -- finish unicode-uint8 +- finish unicode-utf8 +- update www.pypy.org, speed.pypy.org (web devs needed) From pypy.commits at gmail.com Sat Jan 6 12:43:34 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 06 Jan 2018 09:43:34 -0800 (PST) Subject: [pypy-commit] extradoc extradoc: Start the people.txt list Message-ID: <5a510ac6.46101c0a.9a25f.74db@mx.google.com> Author: Armin Rigo Branch: extradoc Changeset: r5859:0ffe7b9497c5 Date: 2018-01-06 18:43 +0100 http://bitbucket.org/pypy/extradoc/changeset/0ffe7b9497c5/ Log: Start the people.txt list diff --git a/sprintinfo/leysin-winter-2018/people.txt b/sprintinfo/leysin-winter-2018/people.txt new file mode 100644 --- /dev/null +++ b/sprintinfo/leysin-winter-2018/people.txt @@ -0,0 +1,26 @@ + +People coming to the Leysin sprint Winter 2018 +================================================== + +People who have a ``?`` in their arrive/depart or accomodation +column are known to be coming but there are no details +available yet from them. + +==================== ============== ======================= + Name Arrive/Depart Accomodation +==================== ============== ======================= +Armin Rigo private +==================== ============== ======================= + +**NOTE:** lodging is by default in Ermina. Based on past years, there +should be two ~4 people rooms as well as smaller rooms on demand. (The +details will become clearer a bit later. Individual rooms should be +available if needed, but ask Armin Rigo.) + +Of course you're free to book a different hotel yourself if you prefer. + +The standard booking is for the nights from Saturday to Saturday, but it +is possible to extend that. If it is possible for you to arrive Sunday +not too late, then you should get a booking from Sunday only. We'll +work from Sunday noon-ish to the next Saturday (with a break day in the +middle of the week). From pypy.commits at gmail.com Sat Jan 6 12:52:04 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 06 Jan 2018 09:52:04 -0800 (PST) Subject: [pypy-commit] extradoc extradoc: add my dates Message-ID: <5a510cc4.410e1c0a.180c4.60d5@mx.google.com> Author: Matti Picus Branch: extradoc Changeset: r5860:7786f522e3fe Date: 2018-01-06 19:51 +0200 http://bitbucket.org/pypy/extradoc/changeset/7786f522e3fe/ Log: add my dates diff --git a/sprintinfo/leysin-winter-2018/people.txt b/sprintinfo/leysin-winter-2018/people.txt --- a/sprintinfo/leysin-winter-2018/people.txt +++ b/sprintinfo/leysin-winter-2018/people.txt @@ -10,6 +10,7 @@ Name Arrive/Depart Accomodation ==================== ============== ======================= Armin Rigo private +Matti Picus 17.3/22.3 Ermina ==================== ============== ======================= **NOTE:** lodging is by default in Ermina. Based on past years, there From pypy.commits at gmail.com Sat Jan 6 17:15:56 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 06 Jan 2018 14:15:56 -0800 (PST) Subject: [pypy-commit] pypy py3-winreg: fix FormatError -> FormatErrorW Message-ID: <5a514a9c.48aadf0a.46301.167d@mx.google.com> Author: Matti Picus Branch: py3-winreg Changeset: r93630:4099cdb5fe33 Date: 2018-01-07 00:14 +0200 http://bitbucket.org/pypy/pypy/changeset/4099cdb5fe33/ Log: fix FormatError -> FormatErrorW diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py --- a/pypy/module/_winreg/interp_winreg.py +++ b/pypy/module/_winreg/interp_winreg.py @@ -8,10 +8,10 @@ from rpython.rlib.rarithmetic import r_uint, intmask def raiseWindowsError(space, errcode, context): - message = rwin32.FormatError(errcode) + message = rwin32.FormatErrorW(errcode) w_errcode = space.newint(errcode) raise OperationError(space.w_WindowsError, - space.newtuple([w_errcode, space.newtext(message), + space.newtuple([w_errcode, space.newunicode(message), space.w_None, w_errcode])) class W_HKEY(W_Root): From pypy.commits at gmail.com Sun Jan 7 16:08:04 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 07 Jan 2018 13:08:04 -0800 (PST) Subject: [pypy-commit] pypy py3.5: fix for issue #2716 Message-ID: <5a528c34.042d1c0a.68c87.1ed9@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93631:0c5a75555005 Date: 2018-01-07 22:27 +0200 http://bitbucket.org/pypy/pypy/changeset/0c5a75555005/ Log: fix for issue #2716 diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py --- a/rpython/rlib/rdynload.py +++ b/rpython/rlib/rdynload.py @@ -228,7 +228,7 @@ res = rwin32.LoadLibrary(name) if not res: err = rwin32.GetLastError_saved() - raise DLOpenError(rwin32.FormatError(err)) + raise DLOpenError(rwin32.FormatErrorW(err)) return res def dlclose(handle): From pypy.commits at gmail.com Sun Jan 7 16:08:11 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 07 Jan 2018 13:08:11 -0800 (PST) Subject: [pypy-commit] pypy py3.5: restart whatsnew-pypy3-head.rst part 1 Message-ID: <5a528c3b.4fc11c0a.731eb.f25a@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93634:2f203c291e34 Date: 2018-01-07 23:02 +0200 http://bitbucket.org/pypy/pypy/changeset/2f203c291e34/ Log: restart whatsnew-pypy3-head.rst part 1 diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-5.10.0.rst rename from pypy/doc/whatsnew-pypy3-head.rst rename to pypy/doc/whatsnew-pypy3-5.10.0.rst From pypy.commits at gmail.com Sun Jan 7 16:08:07 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 07 Jan 2018 13:08:07 -0800 (PST) Subject: [pypy-commit] pypy py3-winreg: close py3-winreg branch to be merged Message-ID: <5a528c37.12711c0a.79577.27ee@mx.google.com> Author: Matti Picus Branch: py3-winreg Changeset: r93632:c4c33df88de7 Date: 2018-01-07 22:28 +0200 http://bitbucket.org/pypy/pypy/changeset/c4c33df88de7/ Log: close py3-winreg branch to be merged From pypy.commits at gmail.com Sun Jan 7 16:08:13 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 07 Jan 2018 13:08:13 -0800 (PST) Subject: [pypy-commit] pypy py3.5: restart whatsnew-pypy3-head.rst part 2 and document merged branches Message-ID: <5a528c3d.94571c0a.a6572.f90f@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93635:15ab2cc81a0c Date: 2018-01-07 23:06 +0200 http://bitbucket.org/pypy/pypy/changeset/15ab2cc81a0c/ Log: restart whatsnew-pypy3-head.rst part 2 and document merged branches diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy3-head.rst @@ -0,0 +1,16 @@ +========================= +What's new in PyPy3 5.10+ +========================= + +.. this is the revision after release-pypy3.5-v5.10 +.. startrev: 34c63fba0bba + +.. branch: hroncok/fix-typeerror-str-does-not-support-the-b-1514414905375 + +Fix for bytestrings in console repl + +.. branch: py3-winreg + +Update winreg module to use unicode, wide-strings + + From pypy.commits at gmail.com Sun Jan 7 16:08:09 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 07 Jan 2018 13:08:09 -0800 (PST) Subject: [pypy-commit] pypy py3.5: merge py3-winreg which uses unicode strings in winreg module Message-ID: <5a528c39.042d1c0a.68c87.1ee0@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93633:7b9edeb55281 Date: 2018-01-07 22:29 +0200 http://bitbucket.org/pypy/pypy/changeset/7b9edeb55281/ Log: merge py3-winreg which uses unicode strings in winreg module diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py --- a/pypy/interpreter/gateway.py +++ b/pypy/interpreter/gateway.py @@ -171,6 +171,9 @@ def visit_text0(self, el, app_sig): self.checked_space_method(el, app_sig) + def visit_unicode(self, el, app_sig): + self.checked_space_method(el, app_sig) + def visit_fsencode(self, el, app_sig): self.checked_space_method(el, app_sig) @@ -320,6 +323,9 @@ def visit_text0(self, typ): self.run_args.append("space.text0_w(%s)" % (self.scopenext(),)) + def visit_unicode(self, typ): + self.run_args.append("space.unicode_w(%s)" % (self.scopenext(),)) + def visit_fsencode(self, typ): self.run_args.append("space.fsencode_w(%s)" % (self.scopenext(),)) @@ -485,6 +491,9 @@ def visit_text(self, typ): self.unwrap.append("space.text_w(%s)" % (self.nextarg(),)) + def visit_unicode(self, typ): + self.unwrap.append("space.unicode_w(%s)" % (self.nextarg(),)) + def visit_text0(self, typ): self.unwrap.append("space.text0_w(%s)" % (self.nextarg(),)) diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py --- a/pypy/module/_winreg/interp_winreg.py +++ b/pypy/module/_winreg/interp_winreg.py @@ -9,9 +9,10 @@ def raiseWindowsError(space, errcode, context): message = rwin32.FormatErrorW(errcode) + w_errcode = space.newint(errcode) raise OperationError(space.w_WindowsError, - space.newtuple([space.newint(errcode), - space.newunicode(message)])) + space.newtuple([w_errcode, space.newunicode(message), + space.w_None, w_errcode])) class W_HKEY(W_Root): def __init__(self, space, hkey): @@ -32,7 +33,7 @@ return space.newint(self.as_int()) def descr_repr(self, space): - return space.newtext("" % (self.as_int(),)) + return space.newunicode(u"" % (self.as_int(),)) def descr_int(self, space): return space.newint(self.as_int()) @@ -149,7 +150,7 @@ if ret != 0: raiseWindowsError(space, ret, 'RegFlushKey') - at unwrap_spec(subkey="text", filename="text") + at unwrap_spec(subkey="unicode", filename="unicode") def LoadKey(space, w_hkey, subkey, filename): """LoadKey(key, sub_key, file_name) - Creates a subkey under the specified key and stores registration information from a specified file into that subkey. @@ -170,11 +171,15 @@ The docs imply key must be in the HKEY_USER or HKEY_LOCAL_MACHINE tree""" # XXX should filename use space.fsencode_w? hkey = hkey_w(w_hkey, space) - ret = rwinreg.RegLoadKey(hkey, subkey, filename) - if ret != 0: - raiseWindowsError(space, ret, 'RegLoadKey') + with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: + c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) + with rffi.scoped_unicode2wcharp(filename) as wide_filename: + c_filename = rffi.cast(rffi.CCHARP, wide_filename) + ret = rwinreg.RegLoadKey(hkey, c_subkey, c_filename) + if ret != 0: + raiseWindowsError(space, ret, 'RegLoadKey') - at unwrap_spec(filename="text") + at unwrap_spec(filename="unicode") def SaveKey(space, w_hkey, filename): """SaveKey(key, file_name) - Saves the specified key, and all its subkeys to the specified file. @@ -189,11 +194,13 @@ The caller of this method must possess the SeBackupPrivilege security privilege. This function passes NULL for security_attributes to the API.""" hkey = hkey_w(w_hkey, space) - ret = rwinreg.RegSaveKey(hkey, filename, None) - if ret != 0: - raiseWindowsError(space, ret, 'RegSaveKey') + with rffi.scoped_unicode2wcharp(filename) as wide_filename: + c_filename = rffi.cast(rffi.CCHARP, wide_filename) + ret = rwinreg.RegSaveKey(hkey, c_filename, None) + if ret != 0: + raiseWindowsError(space, ret, 'RegSaveKey') - at unwrap_spec(typ=int, value="text") + at unwrap_spec(typ=int, value="unicode") def SetValue(space, w_hkey, w_subkey, typ, value): """SetValue(key, sub_key, type, value) - Associates a value with a specified key. @@ -215,14 +222,14 @@ if typ != rwinreg.REG_SZ: raise oefmt(space.w_ValueError, "Type must be winreg.REG_SZ") hkey = hkey_w(w_hkey, space) - if space.is_w(w_subkey, space.w_None): - subkey = None - else: - subkey = space.text_w(w_subkey) - with rffi.scoped_str2charp(value) as dataptr: - ret = rwinreg.RegSetValue(hkey, subkey, rwinreg.REG_SZ, dataptr, len(value)) - if ret != 0: - raiseWindowsError(space, ret, 'RegSetValue') + with rffi.scoped_unicode2wcharp(space.unicode_w(w_subkey)) as subkey: + c_subkey = rffi.cast(rffi.CCHARP, subkey) + with rffi.scoped_unicode2wcharp(value) as dataptr: + c_dataptr = rffi.cast(rffi.CCHARP, dataptr) + ret = rwinreg.RegSetValue(hkey, c_subkey, rwinreg.REG_SZ, + c_dataptr, len(value)) + if ret != 0: + raiseWindowsError(space, ret, 'RegSetValue') def QueryValue(space, w_hkey, w_subkey): """string = QueryValue(key, sub_key) - retrieves the unnamed value for a key. @@ -239,30 +246,37 @@ if space.is_w(w_subkey, space.w_None): subkey = None else: - subkey = space.text_w(w_subkey) - with lltype.scoped_alloc(rwin32.PLONG.TO, 1) as bufsize_p: - ret = rwinreg.RegQueryValue(hkey, subkey, None, bufsize_p) - bufSize = intmask(bufsize_p[0]) - if ret == rwinreg.ERROR_MORE_DATA: - bufSize = 256 - elif ret != 0: - raiseWindowsError(space, ret, 'RegQueryValue') + subkey = space.unicode_w(w_subkey) + with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: + c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) + with lltype.scoped_alloc(rwin32.PLONG.TO, 1) as bufsize_p: + ret = rwinreg.RegQueryValue(hkey, c_subkey, None, bufsize_p) + bufSize = intmask(bufsize_p[0]) + if ret == rwinreg.ERROR_MORE_DATA: + bufSize = 256 + elif ret != 0: + raiseWindowsError(space, ret, 'RegQueryValue') - while True: - with lltype.scoped_alloc(rffi.CCHARP.TO, bufSize) as buf: - ret = rwinreg.RegQueryValue(hkey, subkey, buf, bufsize_p) - if ret == rwinreg.ERROR_MORE_DATA: - # Resize and retry - bufSize *= 2 - bufsize_p[0] = bufSize - continue + while True: + with lltype.scoped_alloc(rffi.CCHARP.TO, bufSize) as buf: + ret = rwinreg.RegQueryValue(hkey, c_subkey, buf, bufsize_p) + if ret == rwinreg.ERROR_MORE_DATA: + print 'bufSize was %d, too small' % bufSize + # Resize and retry + bufSize *= 2 + bufsize_p[0] = bufSize + continue - if ret != 0: - raiseWindowsError(space, ret, 'RegQueryValue') - length = intmask(bufsize_p[0] - 1) - return space.newtext(rffi.charp2strn(buf, length)) + if ret != 0: + raiseWindowsError(space, ret, 'RegQueryValue') + length = intmask(bufsize_p[0] - 1) / 2 + wide_buf = rffi.cast(rffi.CWCHARP, buf) + return space.newunicode(rffi.wcharp2unicoden(wide_buf, length)) def convert_to_regdata(space, w_value, typ): + ''' + returns CCHARP, int + ''' buf = None if typ == rwinreg.REG_DWORD: @@ -282,11 +296,9 @@ buf = lltype.malloc(rffi.CCHARP.TO, buflen, flavor='raw') buf[0] = '\0' else: - if space.isinstance_w(w_value, space.w_unicode): - w_value = space.call_method(w_value, 'encode', - space.newtext('mbcs')) - buf = rffi.str2charp(space.text_w(w_value)) - buflen = space.len_w(w_value) + 1 + buf = rffi.unicode2wcharp(space.unicode_w(w_value)) + buf = rffi.cast(rffi.CCHARP, buf) + buflen = (space.len_w(w_value) * 2) + 1 elif typ == rwinreg.REG_MULTI_SZ: if space.is_w(w_value, space.w_None): @@ -302,28 +314,29 @@ while True: try: w_item = space.next(w_iter) - if space.isinstance_w(w_item, space.w_unicode): - w_item = space.call_method(w_item, 'encode', - space.newtext('mbcs')) - item = space.bytes_w(w_item) + item = space.unicode_w(w_item) strings.append(item) - buflen += len(item) + 1 + buflen += 2 * (len(item) + 1) except OperationError as e: if not e.match(space, space.w_StopIteration): raise # re-raise other app-level exceptions break - buflen += 1 + buflen += 2 buf = lltype.malloc(rffi.CCHARP.TO, buflen, flavor='raw') # Now copy data buflen = 0 for string in strings: - for i in range(len(string)): - buf[buflen + i] = string[i] - buflen += len(string) + 1 + with rffi.scoped_unicode2wcharp(string) as wchr: + c_str = rffi.cast(rffi.CCHARP, wchr) + for i in range(len(string) * 2): + buf[buflen + i] = c_str[i] + buflen += (len(string) + 1) * 2 buf[buflen - 1] = '\0' - buflen += 1 + buf[buflen - 2] = '\0' + buflen += 2 buf[buflen - 1] = '\0' + buf[buflen - 2] = '\0' else: # REG_BINARY and ALL unknown data types. if space.is_w(w_value, space.w_None): @@ -357,37 +370,40 @@ elif typ == rwinreg.REG_SZ or typ == rwinreg.REG_EXPAND_SZ: if not buflen: - s = "" + s = u"" else: # may or may not have a trailing NULL in the buffer. - buf = rffi.cast(rffi.CCHARP, buf) + buf = rffi.cast(rffi.CWCHARP, buf) + buflen /= 2 if buf[buflen - 1] == '\x00': buflen -= 1 - s = rffi.charp2strn(buf, buflen) - w_s = space.newbytes(s) - return space.call_method(w_s, 'decode', space.newtext('mbcs')) + s = rffi.wcharp2unicoden(buf, buflen) + w_s = space.newunicode(s) + return w_s elif typ == rwinreg.REG_MULTI_SZ: if not buflen: return space.newlist([]) + buf = rffi.cast(rffi.CWCHARP, buf) + buflen /= 2 i = 0 l = [] while i < buflen and buf[i]: s = [] - while i < buflen and buf[i] != '\0': + while i < buflen and buf[i] != u'\0': s.append(buf[i]) i += 1 if len(s) == 0: break - s = ''.join(s) - l.append(space.newtext(s)) + s = u''.join(s) + l.append(space.newunicode(s)) i += 1 return space.newlist(l) else: # REG_BINARY and all other types return space.newbytes(rffi.charpsize2str(buf, buflen)) - at unwrap_spec(value_name="text", typ=int) + at unwrap_spec(value_name="unicode", typ=int) def SetValueEx(space, w_hkey, value_name, w_reserved, typ, w_value): """SetValueEx(key, value_name, reserved, type, value) - Stores data in the value field of an open registry key. @@ -422,7 +438,9 @@ hkey = hkey_w(w_hkey, space) buf, buflen = convert_to_regdata(space, w_value, typ) try: - ret = rwinreg.RegSetValueEx(hkey, value_name, 0, typ, buf, buflen) + with rffi.scoped_unicode2wcharp(value_name) as wide_vn: + c_vn = rffi.cast(rffi.CCHARP, wide_vn) + ret = rwinreg.RegSetValueEx(hkey, c_vn, 0, typ, buf, buflen) finally: lltype.free(buf, flavor='raw') if ret != 0: @@ -437,38 +455,40 @@ if space.is_w(w_subkey, space.w_None): subkey = None else: - subkey = space.text_w(w_subkey) + subkey = space.unicode_w(w_subkey) null_dword = lltype.nullptr(rwin32.LPDWORD.TO) - with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retDataSize: - ret = rwinreg.RegQueryValueEx(hkey, subkey, null_dword, null_dword, - None, retDataSize) - bufSize = intmask(retDataSize[0]) - if ret == rwinreg.ERROR_MORE_DATA: - bufSize = 256 - elif ret != 0: - raiseWindowsError(space, ret, 'RegQueryValueEx') + with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: + c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) + with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retDataSize: + ret = rwinreg.RegQueryValueEx(hkey, c_subkey, null_dword, null_dword, + None, retDataSize) + bufSize = intmask(retDataSize[0]) + if ret == rwinreg.ERROR_MORE_DATA: + bufSize = 256 + elif ret != 0: + raiseWindowsError(space, ret, 'RegQueryValueEx') - while True: - with lltype.scoped_alloc(rffi.CCHARP.TO, bufSize) as databuf: - with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retType: + while True: + with lltype.scoped_alloc(rffi.CCHARP.TO, bufSize) as databuf: + with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retType: - ret = rwinreg.RegQueryValueEx(hkey, subkey, null_dword, - retType, databuf, retDataSize) - if ret == rwinreg.ERROR_MORE_DATA: - # Resize and retry - bufSize *= 2 - retDataSize[0] = rffi.cast(rwin32.DWORD, bufSize) - continue - if ret != 0: - raiseWindowsError(space, ret, 'RegQueryValueEx') - length = intmask(retDataSize[0]) - return space.newtuple([ - convert_from_regdata(space, databuf, - length, retType[0]), - space.newint(intmask(retType[0])), + ret = rwinreg.RegQueryValueEx(hkey, c_subkey, null_dword, + retType, databuf, retDataSize) + if ret == rwinreg.ERROR_MORE_DATA: + # Resize and retry + bufSize *= 2 + retDataSize[0] = rffi.cast(rwin32.DWORD, bufSize) + continue + if ret != 0: + raiseWindowsError(space, ret, 'RegQueryValueEx') + length = intmask(retDataSize[0]) + return space.newtuple([ + convert_from_regdata(space, databuf, + length, retType[0]), + space.newint(intmask(retType[0])), ]) - at unwrap_spec(subkey="text") + at unwrap_spec(subkey="unicode") def CreateKey(space, w_hkey, subkey): """key = CreateKey(key, sub_key) - Creates or opens the specified key. @@ -482,13 +502,15 @@ The return value is the handle of the opened key. If the function fails, an exception is raised.""" hkey = hkey_w(w_hkey, space) - with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: - ret = rwinreg.RegCreateKey(hkey, subkey, rethkey) - if ret != 0: - raiseWindowsError(space, ret, 'CreateKey') - return W_HKEY(space, rethkey[0]) + with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: + c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) + with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: + ret = rwinreg.RegCreateKey(hkey, c_subkey, rethkey) + if ret != 0: + raiseWindowsError(space, ret, 'CreateKey') + return W_HKEY(space, rethkey[0]) - at unwrap_spec(sub_key="text", reserved=int, access=rffi.r_uint) + at unwrap_spec(sub_key="unicode", reserved=int, access=rffi.r_uint) def CreateKeyEx(space, w_key, sub_key, reserved=0, access=rwinreg.KEY_WRITE): """key = CreateKey(key, sub_key) - Creates or opens the specified key. @@ -502,17 +524,19 @@ The return value is the handle of the opened key. If the function fails, an exception is raised.""" hkey = hkey_w(w_key, space) - with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: - ret = rwinreg.RegCreateKeyEx(hkey, sub_key, reserved, None, 0, - access, None, rethkey, - lltype.nullptr(rwin32.LPDWORD.TO)) - if ret != 0: - raiseWindowsError(space, ret, 'CreateKeyEx') - return W_HKEY(space, rethkey[0]) + with rffi.scoped_unicode2wcharp(sub_key) as wide_sub_key: + c_subkey = rffi.cast(rffi.CCHARP, wide_sub_key) + with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: + ret = rwinreg.RegCreateKeyEx(hkey, c_subkey, reserved, None, 0, + access, None, rethkey, + lltype.nullptr(rwin32.LPDWORD.TO)) + if ret != 0: + raiseWindowsError(space, ret, 'CreateKeyEx') + return W_HKEY(space, rethkey[0]) - at unwrap_spec(subkey="text") + at unwrap_spec(subkey="unicode") def DeleteKey(space, w_hkey, subkey): - """DeleteKey(key, sub_key) - Deletes the specified key. + """DeleteKey(key, subkey) - Deletes the specified key. key is an already open key, or any one of the predefined HKEY_* constants. sub_key is a string that must be a subkey of the key identified by the key parameter. @@ -523,22 +547,26 @@ If the method succeeds, the entire key, including all of its values, is removed. If the method fails, an EnvironmentError exception is raised.""" hkey = hkey_w(w_hkey, space) - ret = rwinreg.RegDeleteKey(hkey, subkey) - if ret != 0: - raiseWindowsError(space, ret, 'RegDeleteKey') + with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: + c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) + ret = rwinreg.RegDeleteKey(hkey, c_subkey) + if ret != 0: + raiseWindowsError(space, ret, 'RegDeleteKey') - at unwrap_spec(subkey="text") + at unwrap_spec(subkey="unicode") def DeleteValue(space, w_hkey, subkey): """DeleteValue(key, value) - Removes a named value from a registry key. key is an already open key, or any one of the predefined HKEY_* constants. value is a string that identifies the value to remove.""" hkey = hkey_w(w_hkey, space) - ret = rwinreg.RegDeleteValue(hkey, subkey) - if ret != 0: - raiseWindowsError(space, ret, 'RegDeleteValue') + with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: + c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) + ret = rwinreg.RegDeleteValue(hkey, c_subkey) + if ret != 0: + raiseWindowsError(space, ret, 'RegDeleteValue') - at unwrap_spec(sub_key="text", reserved=int, access=rffi.r_uint) + at unwrap_spec(sub_key="unicode", reserved=int, access=rffi.r_uint) def OpenKey(space, w_key, sub_key, reserved=0, access=rwinreg.KEY_READ): """key = OpenKey(key, sub_key, res = 0, sam = KEY_READ) - Opens the specified key. @@ -551,11 +579,13 @@ The result is a new handle to the specified key If the function fails, an EnvironmentError exception is raised.""" hkey = hkey_w(w_key, space) - with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: - ret = rwinreg.RegOpenKeyEx(hkey, sub_key, reserved, access, rethkey) - if ret != 0: - raiseWindowsError(space, ret, 'RegOpenKeyEx') - return W_HKEY(space, rethkey[0]) + with rffi.scoped_unicode2wcharp(sub_key) as wide_subkey: + c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) + with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: + ret = rwinreg.RegOpenKeyEx(hkey, c_subkey, reserved, access, rethkey) + if ret != 0: + raiseWindowsError(space, ret, 'RegOpenKeyEx') + return W_HKEY(space, rethkey[0]) @unwrap_spec(index=int) def EnumValue(space, w_hkey, index): @@ -590,15 +620,16 @@ bufDataSize = intmask(retDataSize[0]) bufValueSize = intmask(retValueSize[0]) - with lltype.scoped_alloc(rffi.CCHARP.TO, + with lltype.scoped_alloc(rffi.CWCHARP.TO, intmask(retValueSize[0])) as valuebuf: while True: with lltype.scoped_alloc(rffi.CCHARP.TO, bufDataSize) as databuf: with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retType: + c_valuebuf = rffi.cast(rffi.CCHARP, valuebuf) ret = rwinreg.RegEnumValue( - hkey, index, valuebuf, retValueSize, + hkey, index, c_valuebuf, retValueSize, null_dword, retType, databuf, retDataSize) if ret == rwinreg.ERROR_MORE_DATA: # Resize and retry @@ -614,7 +645,7 @@ length = intmask(retDataSize[0]) return space.newtuple([ - space.newtext(rffi.charp2str(valuebuf)), + space.newunicode(rffi.wcharp2unicode(valuebuf)), convert_from_regdata(space, databuf, length, retType[0]), space.newint(intmask(retType[0])), @@ -647,7 +678,7 @@ lltype.nullptr(rwin32.PFILETIME.TO)) if ret != 0: raiseWindowsError(space, ret, 'RegEnumKeyEx') - return space.newtext(rffi.charp2str(buf)) + return space.newunicode(rffi.wcharp2unicode(rffi.cast(rffi.CWCHARP, buf))) def QueryInfoKey(space, w_hkey): """tuple = QueryInfoKey(key) - Returns information about a key. @@ -726,7 +757,7 @@ raise oefmt(space.w_NotImplementedError, "not implemented on this platform") - at unwrap_spec(sub_key="text", reserved=int, access=rffi.r_uint) + at unwrap_spec(sub_key="unicode", reserved=int, access=rffi.r_uint) def DeleteKeyEx(space, w_key, sub_key, reserved=0, access=rwinreg.KEY_WOW64_64KEY): """DeleteKeyEx(key, sub_key, sam, res) - Deletes the specified key. diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py --- a/pypy/module/_winreg/test/test_winreg.py +++ b/pypy/module/_winreg/test/test_winreg.py @@ -19,7 +19,7 @@ canSaveKey = len(ret) > 0 class AppTestHKey: - #spaceconfig = dict(usemodules=('_winreg',)) + spaceconfig = dict(usemodules=('_winreg',)) def test_repr(self): import winreg @@ -27,12 +27,12 @@ assert str(k) == "" class AppTestFfi: - #spaceconfig = dict(usemodules=('_winreg',)) + spaceconfig = dict(usemodules=('_winreg',)) def setup_class(cls): - import _winreg + import _winreg as winreg space = cls.space - cls.root_key = _winreg.HKEY_CURRENT_USER + cls.root_key = winreg.HKEY_CURRENT_USER cls.test_key_name = "SOFTWARE\\Pypy Registry Test Key - Delete Me" cls.w_root_key = space.wrap(cls.root_key) cls.w_test_key_name = space.wrap(cls.test_key_name) @@ -40,22 +40,22 @@ cls.w_tmpfilename = space.wrap(str(udir.join('winreg-temp'))) test_data = [ - ("Int Value", 0xFEDCBA98, _winreg.REG_DWORD), - ("Str Value", "A string Value", _winreg.REG_SZ), - ("Unicode Value", u"A unicode Value", _winreg.REG_SZ), - ("Str Expand", "The path is %path%", _winreg.REG_EXPAND_SZ), - ("Multi Str", ["Several", "string", u"values"], _winreg.REG_MULTI_SZ), + ("Int Value", 0xFEDCBA98, winreg.REG_DWORD), + ("Str Value", b"A string Value", winreg.REG_SZ), + ("Unicode Value", "A unicode Value", winreg.REG_SZ), + ("Str Expand", "The path is %path%", winreg.REG_EXPAND_SZ), + ("Multi Str", [b"Several", u"string", u"values"], winreg.REG_MULTI_SZ), ] cls.w_test_data = w_test_data = space.wrap(test_data) w_btest = space.newtuple([space.wrap("Raw data"), space.newbytes("binary\x00data"), - space.wrap(_winreg.REG_BINARY)]) + space.wrap(winreg.REG_BINARY)]) w_test_data.append(w_btest) def teardown_class(cls): - import winreg + import _winreg try: - winreg.DeleteKey(cls.root_key, cls.test_key_name) + _winreg.DeleteKey(cls.root_key, cls.test_key_name) except WindowsError: pass @@ -67,14 +67,14 @@ def test_simple_write(self): from winreg import SetValue, QueryValue, REG_SZ - value = "Some Default value" + value = u"Some Default value" SetValue(self.root_key, self.test_key_name, REG_SZ, value) assert QueryValue(self.root_key, self.test_key_name) == value def test_CreateKey(self): from winreg import CreateKey, QueryInfoKey key = CreateKey(self.root_key, self.test_key_name) - sub_key = CreateKey(key, "sub_key") + sub_key = CreateKey(key, u"sub_key") nkeys, nvalues, since_mod = QueryInfoKey(key) assert nkeys == 1 @@ -86,7 +86,7 @@ from winreg import CreateKeyEx, QueryInfoKey from winreg import KEY_ALL_ACCESS, KEY_READ key = CreateKeyEx(self.root_key, self.test_key_name, 0, KEY_ALL_ACCESS) - sub_key = CreateKeyEx(key, "sub_key", 0, KEY_READ) + sub_key = CreateKeyEx(key, u"sub_key", 0, KEY_READ) nkeys, nvalues, since_mod = QueryInfoKey(key) assert nkeys == 1 @@ -97,7 +97,7 @@ def test_close(self): from winreg import OpenKey, CloseKey, FlushKey, QueryInfoKey key = OpenKey(self.root_key, self.test_key_name) - sub_key = OpenKey(key, "sub_key") + sub_key = OpenKey(key, u"sub_key") int_sub_key = int(sub_key) FlushKey(sub_key) @@ -119,7 +119,7 @@ def test_with(self): from winreg import OpenKey with OpenKey(self.root_key, self.test_key_name) as key: - with OpenKey(key, "sub_key") as sub_key: + with OpenKey(key, u"sub_key") as sub_key: assert key.handle != 0 assert sub_key.handle != 0 assert key.handle == 0 @@ -140,17 +140,17 @@ assert 0, "Did not raise" def test_SetValueEx(self): + # this test leaves open keys. If it fails, others will too from winreg import CreateKey, SetValueEx, REG_BINARY, REG_DWORD key = CreateKey(self.root_key, self.test_key_name) - sub_key = CreateKey(key, "sub_key") + sub_key = CreateKey(key, u"sub_key") SetValueEx(sub_key, 'Int Value', 0, REG_DWORD, None) SetValueEx(sub_key, 'Int Value', 0, REG_DWORD, 45) for name, value, type in self.test_data: SetValueEx(sub_key, name, 0, type, value) - exc = raises(TypeError, SetValueEx, sub_key, 'test_name', None, - REG_BINARY, memoryview('abc')) - assert str(exc.value) == ("Objects of type 'memoryview' can not " - "be used as binary registry values") + # cannot wrap a memoryview in setup_class for test_data + SetValueEx(sub_key, u'test_name', None, + REG_BINARY, memoryview(b'abc')) def test_readValues(self): from winreg import OpenKey, EnumValue, QueryValueEx, EnumKey @@ -163,27 +163,31 @@ data = EnumValue(sub_key, index) except EnvironmentError as e: break - assert data in self.test_data + if data[0] != 'test_name': + # cannot wrap a memoryview in setup_class for test_data + assert data in self.test_data index = index + 1 - assert index == len(self.test_data) + assert index == len(self.test_data) + 1 for name, value, type in self.test_data: result = QueryValueEx(sub_key, name) assert result == (value, type) if type == REG_SZ or type == REG_EXPAND_SZ: - assert isinstance(result[0], unicode) # not string + assert not isinstance(result[0], bytes) assert EnumKey(key, 0) == "sub_key" raises(EnvironmentError, EnumKey, key, 1) def test_delete(self): # must be run after test_SetValueEx - from _winreg import OpenKey, KEY_ALL_ACCESS, DeleteValue, DeleteKey + from winreg import OpenKey, KEY_ALL_ACCESS, DeleteValue, DeleteKey key = OpenKey(self.root_key, self.test_key_name, 0, KEY_ALL_ACCESS) sub_key = OpenKey(key, "sub_key", 0, KEY_ALL_ACCESS) for name, value, type in self.test_data: DeleteValue(sub_key, name) + # cannot wrap a memoryview in setup_class for test_data + DeleteValue(sub_key, 'test_name') DeleteKey(key, "sub_key") @@ -242,7 +246,7 @@ skip("access denied to registry key " "(are you running in a non-interactive session?)") raise - QueryValueEx(HKEY_PERFORMANCE_DATA, None) + QueryValueEx(HKEY_PERFORMANCE_DATA, 'Global') def test_reflection_unsupported(self): import sys diff --git a/rpython/rlib/rwinreg.py b/rpython/rlib/rwinreg.py --- a/rpython/rlib/rwinreg.py +++ b/rpython/rlib/rwinreg.py @@ -47,7 +47,7 @@ HKEY = rwin32.HANDLE PHKEY = rffi.CArrayPtr(HKEY) REGSAM = rwin32.DWORD -suffix = 'A' +suffix = 'W' RegSetValue = external( 'RegSetValue' + suffix, From pypy.commits at gmail.com Mon Jan 8 01:10:19 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 07 Jan 2018 22:10:19 -0800 (PST) Subject: [pypy-commit] pypy py3.5: fix translation? Message-ID: <5a530b4b.09bbdf0a.26336.716f@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93636:29875e4930fc Date: 2018-01-08 08:09 +0200 http://bitbucket.org/pypy/pypy/changeset/29875e4930fc/ Log: fix translation? diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -190,7 +190,7 @@ assert isinstance(w_objclass, W_TypeObject) raise oefmt(space.w_TypeError, "descriptor '%8' of '%s' object needs an argument", - self.name, w_objclass.name) + self.name, self.w_objclass.getname(space)) w_instance = __args__.arguments_w[0] # XXX: needs a stricter test if not space.isinstance_w(w_instance, self.w_objclass): @@ -230,7 +230,7 @@ def descr_call(self, space, __args__): if len(__args__.arguments_w) == 0: raise oefmt(space.w_TypeError, - "descriptor '%s' of '%s' object needs an argument", + "descriptor '%8' of '%s' object needs an argument", self.name, self.w_objclass.getname(space)) w_instance = __args__.arguments_w[0] # XXX typecheck missing # CCC: we can surely do better than this From pypy.commits at gmail.com Mon Jan 8 04:24:52 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 08 Jan 2018 01:24:52 -0800 (PST) Subject: [pypy-commit] pypy py3.5: fix 0c5a75555005, DLOpenError msg must be str not unicode Message-ID: <5a5338e4.59451c0a.9cd14.3004@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93637:caa299dc43a2 Date: 2018-01-08 11:24 +0200 http://bitbucket.org/pypy/pypy/changeset/caa299dc43a2/ Log: fix 0c5a75555005, DLOpenError msg must be str not unicode diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py --- a/pypy/module/_rawffi/interp_rawffi.py +++ b/pypy/module/_rawffi/interp_rawffi.py @@ -154,7 +154,7 @@ msg = e.msg if we_are_translated() else repr(e.msg) else: msg = 'unspecified error' - return oefmt(space.w_OSError, 'Cannot load library %s: %s', filename, msg) + return oefmt(space.w_OSError, 'Cannot load library %s: %8', filename, msg) class W_CDLL(W_Root): diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py --- a/rpython/rlib/rdynload.py +++ b/rpython/rlib/rdynload.py @@ -228,7 +228,9 @@ res = rwin32.LoadLibrary(name) if not res: err = rwin32.GetLastError_saved() - raise DLOpenError(rwin32.FormatErrorW(err)) + ustr = rwin32.FormatErrorW(err) + # DLOpenError unicode msg breaks translation of cpyext create_extension_module + raise DLOpenError(ustr.encode('utf-8')) return res def dlclose(handle): From pypy.commits at gmail.com Mon Jan 8 05:04:08 2018 From: pypy.commits at gmail.com (arigo) Date: Mon, 08 Jan 2018 02:04:08 -0800 (PST) Subject: [pypy-commit] cffi default: Issue #350: more mess on Windows with Py_LIMITED_API Message-ID: <5a534218.85631c0a.3f69b.7697@mx.google.com> Author: Armin Rigo Branch: Changeset: r3054:164e526a5515 Date: 2018-01-08 11:03 +0100 http://bitbucket.org/cffi/cffi/changeset/164e526a5515/ Log: Issue #350: more mess on Windows with Py_LIMITED_API diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h --- a/cffi/_cffi_include.h +++ b/cffi/_cffi_include.h @@ -7,11 +7,38 @@ we can learn about Py_DEBUG from pyconfig.h, but it is unclear if the same works for the other two macros. Py_DEBUG implies them, but not the other way around. + + Issue #350: more mess: on Windows, we have to define Py_LIMITED_API + even before including pyconfig.h. In that case, we guess what + pyconfig.h will do to the macros above, and check our guess after + the #include. */ #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) -# include -# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) -# define Py_LIMITED_API +# ifdef _WIN32 +# if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API +# endif +# include + /* sanity-check: Py_LIMITED_API will cause crashes if any of these + are also defined. Normally, the Python file PC/pyconfig.h does not + cause any of these to be defined, with the exception that _DEBUG + causes Py_DEBUG. Double-check that. */ +# ifdef Py_LIMITED_API +# if defined(Py_DEBUG) +# error "pyconfig.h unexpectedly defines Py_DEBUG but _DEBUG is not set" +# endif +# if defined(Py_TRACE_REFS) +# error "pyconfig.h unexpectedly defines Py_TRACE_REFS" +# endif +# if defined(Py_REF_DEBUG) +# error "pyconfig.h unexpectedly defines Py_REF_DEBUG" +# endif +# endif +# else +# include +# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API +# endif # endif #endif From pypy.commits at gmail.com Mon Jan 8 05:17:10 2018 From: pypy.commits at gmail.com (arigo) Date: Mon, 08 Jan 2018 02:17:10 -0800 (PST) Subject: [pypy-commit] cffi default: Only do that dance if _MSC_VER is defined Message-ID: <5a534526.cf0e1c0a.30f9d.4ad2@mx.google.com> Author: Armin Rigo Branch: Changeset: r3055:14ce6985e1c3 Date: 2018-01-08 11:17 +0100 http://bitbucket.org/cffi/cffi/changeset/14ce6985e1c3/ Log: Only do that dance if _MSC_VER is defined diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h --- a/cffi/_cffi_include.h +++ b/cffi/_cffi_include.h @@ -8,13 +8,13 @@ the same works for the other two macros. Py_DEBUG implies them, but not the other way around. - Issue #350: more mess: on Windows, we have to define Py_LIMITED_API - even before including pyconfig.h. In that case, we guess what - pyconfig.h will do to the macros above, and check our guess after - the #include. + Issue #350: more mess: on Windows, with _MSC_VER, we have to define + Py_LIMITED_API even before including pyconfig.h. In that case, we + guess what pyconfig.h will do to the macros above, and check our + guess after the #include. */ #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) -# ifdef _WIN32 +# ifdef _MSC_VER # if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) # define Py_LIMITED_API # endif From pypy.commits at gmail.com Mon Jan 8 05:25:41 2018 From: pypy.commits at gmail.com (arigo) Date: Mon, 08 Jan 2018 02:25:41 -0800 (PST) Subject: [pypy-commit] extradoc extradoc: Leysin sprint announcement Message-ID: <5a534725.841a1c0a.68212.5010@mx.google.com> Author: Armin Rigo Branch: extradoc Changeset: r5861:b6caaafaea26 Date: 2018-01-08 11:25 +0100 http://bitbucket.org/pypy/extradoc/changeset/b6caaafaea26/ Log: Leysin sprint announcement diff --git a/sprintinfo/leysin-winter-2018/announcement.txt b/sprintinfo/leysin-winter-2018/announcement.txt new file mode 100644 --- /dev/null +++ b/sprintinfo/leysin-winter-2018/announcement.txt @@ -0,0 +1,80 @@ +===================================================================== + PyPy Leysin Winter Sprint (17th - 24th March 2018) +===================================================================== + +The next PyPy sprint will be in Leysin, Switzerland, for the thirteenth +time. This is a fully public sprint: newcomers and topics other than +those proposed below are welcome. + +(Note: this sprint is independent from the suggested April-May sprint in +Poland.) + + +------------------------------ +Goals and topics of the sprint +------------------------------ + +The list of topics is open, but here is our current list: + +- cffi tutorial/overview rewrite +- py3 test runners are too complicated +- make win32 builds green +- make packaging more like cpython/portable builds +- get CI builders for PyPy into mainstream projects (Numpy, Scipy, lxml, uwsgi) +- get more of scientific stack working (tensorflow?) +- cpyext performance improvements +- General 3.5 and 3.6 improvements +- JIT topics: guard-compatible, and the subsequent research project to save and reuse traces across processes +- finish unicode-utf8 +- update www.pypy.org, speed.pypy.org (web devs needed) + +As usual, the main side goal is to have fun in winter sports :-) +We can take a day off (for ski or anything else). + +----------- +Exact times +----------- + +Work days: starting March 18th (~noon), ending March 24th (~noon). + +I have pre-booked the week from Saturday March 17th to Saturday March +24th. If it is possible for you to arrive Sunday before mid-afternoon, +then you can get a booking from Sunday only. The break day should be +around Wednesday. + +It is fine to stay a few more days on either side, or conversely to book +for a part of that time only. + +----------------------- +Location & Accomodation +----------------------- + +Leysin, Switzerland, "same place as before". Let me refresh your +memory: both the sprint venue and the lodging will be in a pair of +chalets built specifically for bed & breakfast: http://www.ermina.ch/. +You can also arrange your own lodging elsewhere (as long as you are in +Leysin, you cannot be more than a 15 minutes walk away from the sprint +venue). + +Please *confirm* that you are coming so that we can adjust the +reservations as appropriate. + +The options of rooms are a bit more limited than on previous years +because the place for bed-and-breakfast is shrinking; but we should +still have enough room for us. The price is around 60 CHF, breakfast +included, in shared rooms (3 or 4 people). If there are people that +would prefer a double or single room, please contact me and we'll see +what choices you have. There is a choice of hotels in Leysin, starting +at 65 CHF the room. + +Please register by Mercurial:: + + https://bitbucket.org/pypy/extradoc/ + https://bitbucket.org/pypy/extradoc/raw/extradoc/sprintinfo/leysin-winter-2018/ + +or on the pypy-dev mailing list if you do not yet have check-in rights: + + http://mail.python.org/mailman/listinfo/pypy-dev + +You need a Swiss-to-(insert country here) power adapter. There will be +some Swiss-to-EU adapters around, and at least one EU-format power strip. From pypy.commits at gmail.com Mon Jan 8 08:55:27 2018 From: pypy.commits at gmail.com (arigo) Date: Mon, 08 Jan 2018 05:55:27 -0800 (PST) Subject: [pypy-commit] cffi doc-set_source: More tweaks Message-ID: <5a53784f.cd5c1c0a.ff3da.f747@mx.google.com> Author: Armin Rigo Branch: doc-set_source Changeset: r3056:d2fac65c0e82 Date: 2018-01-08 14:55 +0100 http://bitbucket.org/cffi/cffi/changeset/d2fac65c0e82/ Log: More tweaks diff --git a/doc/source/overview.rst b/doc/source/overview.rst --- a/doc/source/overview.rst +++ b/doc/source/overview.rst @@ -45,8 +45,9 @@ there, %s!\n"``. In general it is ``somestring.encode(myencoding)``. *Python 3 on Windows:* ``ffi.dlopen(None)`` does not work. This problem -is messy and not really fixable. The example above could be fixed by -calling another function from a specific DLL that exists on your system. +is messy and not really fixable. The problem does not occur if you try +to call a fucntion from a specific DLL that exists on your system: then +you use ``ffi.dlopen("path.dll")``. *This example does not call any C compiler. It works in the so-called ABI mode, which means that it will crash if you call some function or @@ -80,8 +81,9 @@ // contains implementation of things declared in cdef() #include #include - struct passwd *mygetpwuid(int uid) { - return getpwuid(uid); + + struct passwd *get_pw_for_root(void) { + return getpwuid(0); } """, libraries=[]) # or a list of libraries to link with @@ -89,12 +91,13 @@ # include_dirs=[..], extra_objects=[..], and so on) ffibuilder.cdef(""" - // declarations that are shared between python and C + // declarations that are shared between Python and C struct passwd { char *pw_name; ...; // literally dot-dot-dot }; - struct passwd *mygetpwuid(int uid); + struct passwd *getpwuid(int uid); // defined in + struct passwd *get_pw_for_root(void); // defined in set_source() """) if __name__ == "__main__": @@ -116,17 +119,20 @@ from _example import ffi, lib - p = lib.mygetpwuid(0) + p = lib.getpwuid(0) + assert ffi.string(p.pw_name) == b'root' + p = lib.get_pw_for_root() assert ffi.string(p.pw_name) == b'root' Note that this works independently of the exact C layout of ``struct passwd`` (it is "API level", as opposed to "ABI level"). It requires a C compiler in order to run ``example_build.py``, but it is much more portable than trying to get the details of the fields of ``struct -passwd`` exactly right. Similarly, we declared ``mygetpwuid()`` as -taking an ``int`` argument. On some platforms this might be slightly -incorrect---but it does not matter. It is also faster than the ABI -mode. +passwd`` exactly right. Similarly, in the ``cdef()`` we declared +``getpwuid()`` as taking an ``int`` argument; on some platforms this +might be slightly incorrect---but it does not matter. + +Note also that at runtime, the API mode is faster than the ABI mode. To integrate it inside a ``setup.py`` distribution with Setuptools: From pypy.commits at gmail.com Mon Jan 8 10:04:56 2018 From: pypy.commits at gmail.com (arigo) Date: Mon, 08 Jan 2018 07:04:56 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Fix for python 3 Message-ID: <5a538898.b0abdf0a.c8456.5bca@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r93638:3352216b0e57 Date: 2018-01-08 16:04 +0100 http://bitbucket.org/pypy/pypy/changeset/3352216b0e57/ Log: Fix for python 3 diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py --- a/lib_pypy/_ctypes/array.py +++ b/lib_pypy/_ctypes/array.py @@ -12,8 +12,7 @@ if cls == (_CData,): # this is the Array class defined below res._ffiarray = None return res - if not hasattr(res, '_length_') or not isinstance(res._length_, - (int, long)): + if not hasattr(res, '_length_') or not isinstance(res._length_, int): raise AttributeError( "class must define a '_length_' attribute, " "which must be a positive integer") From pypy.commits at gmail.com Mon Jan 8 13:46:44 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 08 Jan 2018 10:46:44 -0800 (PST) Subject: [pypy-commit] pypy release-pypy3.5-v5.9.x: merge all but merge-from-default from py3.5 into release Message-ID: <5a53bc94.05c0df0a.d86f9.4411@mx.google.com> Author: Matti Picus Branch: release-pypy3.5-v5.9.x Changeset: r93639:e2d43f9eb080 Date: 2018-01-08 19:59 +0200 http://bitbucket.org/pypy/pypy/changeset/e2d43f9eb080/ Log: merge all but merge-from-default from py3.5 into release diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py --- a/pypy/interpreter/gateway.py +++ b/pypy/interpreter/gateway.py @@ -171,6 +171,9 @@ def visit_text0(self, el, app_sig): self.checked_space_method(el, app_sig) + def visit_unicode(self, el, app_sig): + self.checked_space_method(el, app_sig) + def visit_fsencode(self, el, app_sig): self.checked_space_method(el, app_sig) @@ -320,6 +323,9 @@ def visit_text0(self, typ): self.run_args.append("space.text0_w(%s)" % (self.scopenext(),)) + def visit_unicode(self, typ): + self.run_args.append("space.unicode_w(%s)" % (self.scopenext(),)) + def visit_fsencode(self, typ): self.run_args.append("space.fsencode_w(%s)" % (self.scopenext(),)) @@ -485,6 +491,9 @@ def visit_text(self, typ): self.unwrap.append("space.text_w(%s)" % (self.nextarg(),)) + def visit_unicode(self, typ): + self.unwrap.append("space.unicode_w(%s)" % (self.nextarg(),)) + def visit_text0(self, typ): self.unwrap.append("space.text0_w(%s)" % (self.nextarg(),)) diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py --- a/pypy/module/_rawffi/interp_rawffi.py +++ b/pypy/module/_rawffi/interp_rawffi.py @@ -154,7 +154,7 @@ msg = e.msg if we_are_translated() else repr(e.msg) else: msg = 'unspecified error' - return oefmt(space.w_OSError, 'Cannot load library %s: %s', filename, msg) + return oefmt(space.w_OSError, 'Cannot load library %s: %8', filename, msg) class W_CDLL(W_Root): diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py --- a/pypy/module/_winreg/interp_winreg.py +++ b/pypy/module/_winreg/interp_winreg.py @@ -9,9 +9,10 @@ def raiseWindowsError(space, errcode, context): message = rwin32.FormatErrorW(errcode) + w_errcode = space.newint(errcode) raise OperationError(space.w_WindowsError, - space.newtuple([space.newint(errcode), - space.newunicode(message)])) + space.newtuple([w_errcode, space.newunicode(message), + space.w_None, w_errcode])) class W_HKEY(W_Root): def __init__(self, space, hkey): @@ -32,7 +33,7 @@ return space.newint(self.as_int()) def descr_repr(self, space): - return space.newtext("" % (self.as_int(),)) + return space.newunicode(u"" % (self.as_int(),)) def descr_int(self, space): return space.newint(self.as_int()) @@ -149,7 +150,7 @@ if ret != 0: raiseWindowsError(space, ret, 'RegFlushKey') - at unwrap_spec(subkey="text", filename="text") + at unwrap_spec(subkey="unicode", filename="unicode") def LoadKey(space, w_hkey, subkey, filename): """LoadKey(key, sub_key, file_name) - Creates a subkey under the specified key and stores registration information from a specified file into that subkey. @@ -170,11 +171,15 @@ The docs imply key must be in the HKEY_USER or HKEY_LOCAL_MACHINE tree""" # XXX should filename use space.fsencode_w? hkey = hkey_w(w_hkey, space) - ret = rwinreg.RegLoadKey(hkey, subkey, filename) - if ret != 0: - raiseWindowsError(space, ret, 'RegLoadKey') + with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: + c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) + with rffi.scoped_unicode2wcharp(filename) as wide_filename: + c_filename = rffi.cast(rffi.CCHARP, wide_filename) + ret = rwinreg.RegLoadKey(hkey, c_subkey, c_filename) + if ret != 0: + raiseWindowsError(space, ret, 'RegLoadKey') - at unwrap_spec(filename="text") + at unwrap_spec(filename="unicode") def SaveKey(space, w_hkey, filename): """SaveKey(key, file_name) - Saves the specified key, and all its subkeys to the specified file. @@ -189,11 +194,13 @@ The caller of this method must possess the SeBackupPrivilege security privilege. This function passes NULL for security_attributes to the API.""" hkey = hkey_w(w_hkey, space) - ret = rwinreg.RegSaveKey(hkey, filename, None) - if ret != 0: - raiseWindowsError(space, ret, 'RegSaveKey') + with rffi.scoped_unicode2wcharp(filename) as wide_filename: + c_filename = rffi.cast(rffi.CCHARP, wide_filename) + ret = rwinreg.RegSaveKey(hkey, c_filename, None) + if ret != 0: + raiseWindowsError(space, ret, 'RegSaveKey') - at unwrap_spec(typ=int, value="text") + at unwrap_spec(typ=int, value="unicode") def SetValue(space, w_hkey, w_subkey, typ, value): """SetValue(key, sub_key, type, value) - Associates a value with a specified key. @@ -215,14 +222,14 @@ if typ != rwinreg.REG_SZ: raise oefmt(space.w_ValueError, "Type must be winreg.REG_SZ") hkey = hkey_w(w_hkey, space) - if space.is_w(w_subkey, space.w_None): - subkey = None - else: - subkey = space.text_w(w_subkey) - with rffi.scoped_str2charp(value) as dataptr: - ret = rwinreg.RegSetValue(hkey, subkey, rwinreg.REG_SZ, dataptr, len(value)) - if ret != 0: - raiseWindowsError(space, ret, 'RegSetValue') + with rffi.scoped_unicode2wcharp(space.unicode_w(w_subkey)) as subkey: + c_subkey = rffi.cast(rffi.CCHARP, subkey) + with rffi.scoped_unicode2wcharp(value) as dataptr: + c_dataptr = rffi.cast(rffi.CCHARP, dataptr) + ret = rwinreg.RegSetValue(hkey, c_subkey, rwinreg.REG_SZ, + c_dataptr, len(value)) + if ret != 0: + raiseWindowsError(space, ret, 'RegSetValue') def QueryValue(space, w_hkey, w_subkey): """string = QueryValue(key, sub_key) - retrieves the unnamed value for a key. @@ -239,30 +246,37 @@ if space.is_w(w_subkey, space.w_None): subkey = None else: - subkey = space.text_w(w_subkey) - with lltype.scoped_alloc(rwin32.PLONG.TO, 1) as bufsize_p: - ret = rwinreg.RegQueryValue(hkey, subkey, None, bufsize_p) - bufSize = intmask(bufsize_p[0]) - if ret == rwinreg.ERROR_MORE_DATA: - bufSize = 256 - elif ret != 0: - raiseWindowsError(space, ret, 'RegQueryValue') + subkey = space.unicode_w(w_subkey) + with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: + c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) + with lltype.scoped_alloc(rwin32.PLONG.TO, 1) as bufsize_p: + ret = rwinreg.RegQueryValue(hkey, c_subkey, None, bufsize_p) + bufSize = intmask(bufsize_p[0]) + if ret == rwinreg.ERROR_MORE_DATA: + bufSize = 256 + elif ret != 0: + raiseWindowsError(space, ret, 'RegQueryValue') - while True: - with lltype.scoped_alloc(rffi.CCHARP.TO, bufSize) as buf: - ret = rwinreg.RegQueryValue(hkey, subkey, buf, bufsize_p) - if ret == rwinreg.ERROR_MORE_DATA: - # Resize and retry - bufSize *= 2 - bufsize_p[0] = bufSize - continue + while True: + with lltype.scoped_alloc(rffi.CCHARP.TO, bufSize) as buf: + ret = rwinreg.RegQueryValue(hkey, c_subkey, buf, bufsize_p) + if ret == rwinreg.ERROR_MORE_DATA: + print 'bufSize was %d, too small' % bufSize + # Resize and retry + bufSize *= 2 + bufsize_p[0] = bufSize + continue - if ret != 0: - raiseWindowsError(space, ret, 'RegQueryValue') - length = intmask(bufsize_p[0] - 1) - return space.newtext(rffi.charp2strn(buf, length)) + if ret != 0: + raiseWindowsError(space, ret, 'RegQueryValue') + length = intmask(bufsize_p[0] - 1) / 2 + wide_buf = rffi.cast(rffi.CWCHARP, buf) + return space.newunicode(rffi.wcharp2unicoden(wide_buf, length)) def convert_to_regdata(space, w_value, typ): + ''' + returns CCHARP, int + ''' buf = None if typ == rwinreg.REG_DWORD: @@ -282,11 +296,9 @@ buf = lltype.malloc(rffi.CCHARP.TO, buflen, flavor='raw') buf[0] = '\0' else: - if space.isinstance_w(w_value, space.w_unicode): - w_value = space.call_method(w_value, 'encode', - space.newtext('mbcs')) - buf = rffi.str2charp(space.text_w(w_value)) - buflen = space.len_w(w_value) + 1 + buf = rffi.unicode2wcharp(space.unicode_w(w_value)) + buf = rffi.cast(rffi.CCHARP, buf) + buflen = (space.len_w(w_value) * 2) + 1 elif typ == rwinreg.REG_MULTI_SZ: if space.is_w(w_value, space.w_None): @@ -302,28 +314,29 @@ while True: try: w_item = space.next(w_iter) - if space.isinstance_w(w_item, space.w_unicode): - w_item = space.call_method(w_item, 'encode', - space.newtext('mbcs')) - item = space.bytes_w(w_item) + item = space.unicode_w(w_item) strings.append(item) - buflen += len(item) + 1 + buflen += 2 * (len(item) + 1) except OperationError as e: if not e.match(space, space.w_StopIteration): raise # re-raise other app-level exceptions break - buflen += 1 + buflen += 2 buf = lltype.malloc(rffi.CCHARP.TO, buflen, flavor='raw') # Now copy data buflen = 0 for string in strings: - for i in range(len(string)): - buf[buflen + i] = string[i] - buflen += len(string) + 1 + with rffi.scoped_unicode2wcharp(string) as wchr: + c_str = rffi.cast(rffi.CCHARP, wchr) + for i in range(len(string) * 2): + buf[buflen + i] = c_str[i] + buflen += (len(string) + 1) * 2 buf[buflen - 1] = '\0' - buflen += 1 + buf[buflen - 2] = '\0' + buflen += 2 buf[buflen - 1] = '\0' + buf[buflen - 2] = '\0' else: # REG_BINARY and ALL unknown data types. if space.is_w(w_value, space.w_None): @@ -357,37 +370,40 @@ elif typ == rwinreg.REG_SZ or typ == rwinreg.REG_EXPAND_SZ: if not buflen: - s = "" + s = u"" else: # may or may not have a trailing NULL in the buffer. - buf = rffi.cast(rffi.CCHARP, buf) + buf = rffi.cast(rffi.CWCHARP, buf) + buflen /= 2 if buf[buflen - 1] == '\x00': buflen -= 1 - s = rffi.charp2strn(buf, buflen) - w_s = space.newbytes(s) - return space.call_method(w_s, 'decode', space.newtext('mbcs')) + s = rffi.wcharp2unicoden(buf, buflen) + w_s = space.newunicode(s) + return w_s elif typ == rwinreg.REG_MULTI_SZ: if not buflen: return space.newlist([]) + buf = rffi.cast(rffi.CWCHARP, buf) + buflen /= 2 i = 0 l = [] while i < buflen and buf[i]: s = [] - while i < buflen and buf[i] != '\0': + while i < buflen and buf[i] != u'\0': s.append(buf[i]) i += 1 if len(s) == 0: break - s = ''.join(s) - l.append(space.newtext(s)) + s = u''.join(s) + l.append(space.newunicode(s)) i += 1 return space.newlist(l) else: # REG_BINARY and all other types return space.newbytes(rffi.charpsize2str(buf, buflen)) - at unwrap_spec(value_name="text", typ=int) + at unwrap_spec(value_name="unicode", typ=int) def SetValueEx(space, w_hkey, value_name, w_reserved, typ, w_value): """SetValueEx(key, value_name, reserved, type, value) - Stores data in the value field of an open registry key. @@ -422,7 +438,9 @@ hkey = hkey_w(w_hkey, space) buf, buflen = convert_to_regdata(space, w_value, typ) try: - ret = rwinreg.RegSetValueEx(hkey, value_name, 0, typ, buf, buflen) + with rffi.scoped_unicode2wcharp(value_name) as wide_vn: + c_vn = rffi.cast(rffi.CCHARP, wide_vn) + ret = rwinreg.RegSetValueEx(hkey, c_vn, 0, typ, buf, buflen) finally: lltype.free(buf, flavor='raw') if ret != 0: @@ -437,38 +455,40 @@ if space.is_w(w_subkey, space.w_None): subkey = None else: - subkey = space.text_w(w_subkey) + subkey = space.unicode_w(w_subkey) null_dword = lltype.nullptr(rwin32.LPDWORD.TO) - with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retDataSize: - ret = rwinreg.RegQueryValueEx(hkey, subkey, null_dword, null_dword, - None, retDataSize) - bufSize = intmask(retDataSize[0]) - if ret == rwinreg.ERROR_MORE_DATA: - bufSize = 256 - elif ret != 0: - raiseWindowsError(space, ret, 'RegQueryValueEx') + with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: + c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) + with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retDataSize: + ret = rwinreg.RegQueryValueEx(hkey, c_subkey, null_dword, null_dword, + None, retDataSize) + bufSize = intmask(retDataSize[0]) + if ret == rwinreg.ERROR_MORE_DATA: + bufSize = 256 + elif ret != 0: + raiseWindowsError(space, ret, 'RegQueryValueEx') - while True: - with lltype.scoped_alloc(rffi.CCHARP.TO, bufSize) as databuf: - with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retType: + while True: + with lltype.scoped_alloc(rffi.CCHARP.TO, bufSize) as databuf: + with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retType: - ret = rwinreg.RegQueryValueEx(hkey, subkey, null_dword, - retType, databuf, retDataSize) - if ret == rwinreg.ERROR_MORE_DATA: - # Resize and retry - bufSize *= 2 - retDataSize[0] = rffi.cast(rwin32.DWORD, bufSize) - continue - if ret != 0: - raiseWindowsError(space, ret, 'RegQueryValueEx') - length = intmask(retDataSize[0]) - return space.newtuple([ - convert_from_regdata(space, databuf, - length, retType[0]), - space.newint(intmask(retType[0])), + ret = rwinreg.RegQueryValueEx(hkey, c_subkey, null_dword, + retType, databuf, retDataSize) + if ret == rwinreg.ERROR_MORE_DATA: + # Resize and retry + bufSize *= 2 + retDataSize[0] = rffi.cast(rwin32.DWORD, bufSize) + continue + if ret != 0: + raiseWindowsError(space, ret, 'RegQueryValueEx') + length = intmask(retDataSize[0]) + return space.newtuple([ + convert_from_regdata(space, databuf, + length, retType[0]), + space.newint(intmask(retType[0])), ]) - at unwrap_spec(subkey="text") + at unwrap_spec(subkey="unicode") def CreateKey(space, w_hkey, subkey): """key = CreateKey(key, sub_key) - Creates or opens the specified key. @@ -482,13 +502,15 @@ The return value is the handle of the opened key. If the function fails, an exception is raised.""" hkey = hkey_w(w_hkey, space) - with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: - ret = rwinreg.RegCreateKey(hkey, subkey, rethkey) - if ret != 0: - raiseWindowsError(space, ret, 'CreateKey') - return W_HKEY(space, rethkey[0]) + with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: + c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) + with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: + ret = rwinreg.RegCreateKey(hkey, c_subkey, rethkey) + if ret != 0: + raiseWindowsError(space, ret, 'CreateKey') + return W_HKEY(space, rethkey[0]) - at unwrap_spec(sub_key="text", reserved=int, access=rffi.r_uint) + at unwrap_spec(sub_key="unicode", reserved=int, access=rffi.r_uint) def CreateKeyEx(space, w_key, sub_key, reserved=0, access=rwinreg.KEY_WRITE): """key = CreateKey(key, sub_key) - Creates or opens the specified key. @@ -502,17 +524,19 @@ The return value is the handle of the opened key. If the function fails, an exception is raised.""" hkey = hkey_w(w_key, space) - with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: - ret = rwinreg.RegCreateKeyEx(hkey, sub_key, reserved, None, 0, - access, None, rethkey, - lltype.nullptr(rwin32.LPDWORD.TO)) - if ret != 0: - raiseWindowsError(space, ret, 'CreateKeyEx') - return W_HKEY(space, rethkey[0]) + with rffi.scoped_unicode2wcharp(sub_key) as wide_sub_key: + c_subkey = rffi.cast(rffi.CCHARP, wide_sub_key) + with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: + ret = rwinreg.RegCreateKeyEx(hkey, c_subkey, reserved, None, 0, + access, None, rethkey, + lltype.nullptr(rwin32.LPDWORD.TO)) + if ret != 0: + raiseWindowsError(space, ret, 'CreateKeyEx') + return W_HKEY(space, rethkey[0]) - at unwrap_spec(subkey="text") + at unwrap_spec(subkey="unicode") def DeleteKey(space, w_hkey, subkey): - """DeleteKey(key, sub_key) - Deletes the specified key. + """DeleteKey(key, subkey) - Deletes the specified key. key is an already open key, or any one of the predefined HKEY_* constants. sub_key is a string that must be a subkey of the key identified by the key parameter. @@ -523,22 +547,26 @@ If the method succeeds, the entire key, including all of its values, is removed. If the method fails, an EnvironmentError exception is raised.""" hkey = hkey_w(w_hkey, space) - ret = rwinreg.RegDeleteKey(hkey, subkey) - if ret != 0: - raiseWindowsError(space, ret, 'RegDeleteKey') + with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: + c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) + ret = rwinreg.RegDeleteKey(hkey, c_subkey) + if ret != 0: + raiseWindowsError(space, ret, 'RegDeleteKey') - at unwrap_spec(subkey="text") + at unwrap_spec(subkey="unicode") def DeleteValue(space, w_hkey, subkey): """DeleteValue(key, value) - Removes a named value from a registry key. key is an already open key, or any one of the predefined HKEY_* constants. value is a string that identifies the value to remove.""" hkey = hkey_w(w_hkey, space) - ret = rwinreg.RegDeleteValue(hkey, subkey) - if ret != 0: - raiseWindowsError(space, ret, 'RegDeleteValue') + with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: + c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) + ret = rwinreg.RegDeleteValue(hkey, c_subkey) + if ret != 0: + raiseWindowsError(space, ret, 'RegDeleteValue') - at unwrap_spec(sub_key="text", reserved=int, access=rffi.r_uint) + at unwrap_spec(sub_key="unicode", reserved=int, access=rffi.r_uint) def OpenKey(space, w_key, sub_key, reserved=0, access=rwinreg.KEY_READ): """key = OpenKey(key, sub_key, res = 0, sam = KEY_READ) - Opens the specified key. @@ -551,11 +579,13 @@ The result is a new handle to the specified key If the function fails, an EnvironmentError exception is raised.""" hkey = hkey_w(w_key, space) - with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: - ret = rwinreg.RegOpenKeyEx(hkey, sub_key, reserved, access, rethkey) - if ret != 0: - raiseWindowsError(space, ret, 'RegOpenKeyEx') - return W_HKEY(space, rethkey[0]) + with rffi.scoped_unicode2wcharp(sub_key) as wide_subkey: + c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) + with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: + ret = rwinreg.RegOpenKeyEx(hkey, c_subkey, reserved, access, rethkey) + if ret != 0: + raiseWindowsError(space, ret, 'RegOpenKeyEx') + return W_HKEY(space, rethkey[0]) @unwrap_spec(index=int) def EnumValue(space, w_hkey, index): @@ -590,15 +620,16 @@ bufDataSize = intmask(retDataSize[0]) bufValueSize = intmask(retValueSize[0]) - with lltype.scoped_alloc(rffi.CCHARP.TO, + with lltype.scoped_alloc(rffi.CWCHARP.TO, intmask(retValueSize[0])) as valuebuf: while True: with lltype.scoped_alloc(rffi.CCHARP.TO, bufDataSize) as databuf: with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retType: + c_valuebuf = rffi.cast(rffi.CCHARP, valuebuf) ret = rwinreg.RegEnumValue( - hkey, index, valuebuf, retValueSize, + hkey, index, c_valuebuf, retValueSize, null_dword, retType, databuf, retDataSize) if ret == rwinreg.ERROR_MORE_DATA: # Resize and retry @@ -614,7 +645,7 @@ length = intmask(retDataSize[0]) return space.newtuple([ - space.newtext(rffi.charp2str(valuebuf)), + space.newunicode(rffi.wcharp2unicode(valuebuf)), convert_from_regdata(space, databuf, length, retType[0]), space.newint(intmask(retType[0])), @@ -647,7 +678,7 @@ lltype.nullptr(rwin32.PFILETIME.TO)) if ret != 0: raiseWindowsError(space, ret, 'RegEnumKeyEx') - return space.newtext(rffi.charp2str(buf)) + return space.newunicode(rffi.wcharp2unicode(rffi.cast(rffi.CWCHARP, buf))) def QueryInfoKey(space, w_hkey): """tuple = QueryInfoKey(key) - Returns information about a key. @@ -726,7 +757,7 @@ raise oefmt(space.w_NotImplementedError, "not implemented on this platform") - at unwrap_spec(sub_key="text", reserved=int, access=rffi.r_uint) + at unwrap_spec(sub_key="unicode", reserved=int, access=rffi.r_uint) def DeleteKeyEx(space, w_key, sub_key, reserved=0, access=rwinreg.KEY_WOW64_64KEY): """DeleteKeyEx(key, sub_key, sam, res) - Deletes the specified key. diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py --- a/pypy/module/_winreg/test/test_winreg.py +++ b/pypy/module/_winreg/test/test_winreg.py @@ -12,14 +12,14 @@ priv_flags = win32security.TOKEN_ADJUST_PRIVILEGES | win32security.TOKEN_QUERY hToken = win32security.OpenProcessToken (win32api.GetCurrentProcess (), priv_flags) privilege_id = win32security.LookupPrivilegeValue (None, "SeBackupPrivilege") - win32security.AdjustTokenPrivileges (hToken, 0, [(privilege_id, win32security.SE_PRIVILEGE_ENABLED)]) + ret = win32security.AdjustTokenPrivileges (hToken, 0, [(privilege_id, win32security.SE_PRIVILEGE_ENABLED)]) except: canSaveKey = False else: - canSaveKey = True + canSaveKey = len(ret) > 0 class AppTestHKey: - #spaceconfig = dict(usemodules=('_winreg',)) + spaceconfig = dict(usemodules=('_winreg',)) def test_repr(self): import winreg @@ -27,12 +27,12 @@ assert str(k) == "" class AppTestFfi: - #spaceconfig = dict(usemodules=('_winreg',)) + spaceconfig = dict(usemodules=('_winreg',)) def setup_class(cls): - import _winreg + import _winreg as winreg space = cls.space - cls.root_key = _winreg.HKEY_CURRENT_USER + cls.root_key = winreg.HKEY_CURRENT_USER cls.test_key_name = "SOFTWARE\\Pypy Registry Test Key - Delete Me" cls.w_root_key = space.wrap(cls.root_key) cls.w_test_key_name = space.wrap(cls.test_key_name) @@ -40,22 +40,22 @@ cls.w_tmpfilename = space.wrap(str(udir.join('winreg-temp'))) test_data = [ - ("Int Value", 0xFEDCBA98, _winreg.REG_DWORD), - ("Str Value", "A string Value", _winreg.REG_SZ), - ("Unicode Value", u"A unicode Value", _winreg.REG_SZ), - ("Str Expand", "The path is %path%", _winreg.REG_EXPAND_SZ), - ("Multi Str", ["Several", "string", u"values"], _winreg.REG_MULTI_SZ), + ("Int Value", 0xFEDCBA98, winreg.REG_DWORD), + ("Str Value", b"A string Value", winreg.REG_SZ), + ("Unicode Value", "A unicode Value", winreg.REG_SZ), + ("Str Expand", "The path is %path%", winreg.REG_EXPAND_SZ), + ("Multi Str", [b"Several", u"string", u"values"], winreg.REG_MULTI_SZ), ] cls.w_test_data = w_test_data = space.wrap(test_data) w_btest = space.newtuple([space.wrap("Raw data"), space.newbytes("binary\x00data"), - space.wrap(_winreg.REG_BINARY)]) + space.wrap(winreg.REG_BINARY)]) w_test_data.append(w_btest) def teardown_class(cls): - import winreg + import _winreg try: - winreg.DeleteKey(cls.root_key, cls.test_key_name) + _winreg.DeleteKey(cls.root_key, cls.test_key_name) except WindowsError: pass @@ -67,14 +67,14 @@ def test_simple_write(self): from winreg import SetValue, QueryValue, REG_SZ - value = "Some Default value" + value = u"Some Default value" SetValue(self.root_key, self.test_key_name, REG_SZ, value) assert QueryValue(self.root_key, self.test_key_name) == value def test_CreateKey(self): from winreg import CreateKey, QueryInfoKey key = CreateKey(self.root_key, self.test_key_name) - sub_key = CreateKey(key, "sub_key") + sub_key = CreateKey(key, u"sub_key") nkeys, nvalues, since_mod = QueryInfoKey(key) assert nkeys == 1 @@ -86,7 +86,7 @@ from winreg import CreateKeyEx, QueryInfoKey from winreg import KEY_ALL_ACCESS, KEY_READ key = CreateKeyEx(self.root_key, self.test_key_name, 0, KEY_ALL_ACCESS) - sub_key = CreateKeyEx(key, "sub_key", 0, KEY_READ) + sub_key = CreateKeyEx(key, u"sub_key", 0, KEY_READ) nkeys, nvalues, since_mod = QueryInfoKey(key) assert nkeys == 1 @@ -97,7 +97,7 @@ def test_close(self): from winreg import OpenKey, CloseKey, FlushKey, QueryInfoKey key = OpenKey(self.root_key, self.test_key_name) - sub_key = OpenKey(key, "sub_key") + sub_key = OpenKey(key, u"sub_key") int_sub_key = int(sub_key) FlushKey(sub_key) @@ -119,7 +119,7 @@ def test_with(self): from winreg import OpenKey with OpenKey(self.root_key, self.test_key_name) as key: - with OpenKey(key, "sub_key") as sub_key: + with OpenKey(key, u"sub_key") as sub_key: assert key.handle != 0 assert sub_key.handle != 0 assert key.handle == 0 @@ -140,17 +140,17 @@ assert 0, "Did not raise" def test_SetValueEx(self): + # this test leaves open keys. If it fails, others will too from winreg import CreateKey, SetValueEx, REG_BINARY, REG_DWORD key = CreateKey(self.root_key, self.test_key_name) - sub_key = CreateKey(key, "sub_key") + sub_key = CreateKey(key, u"sub_key") SetValueEx(sub_key, 'Int Value', 0, REG_DWORD, None) SetValueEx(sub_key, 'Int Value', 0, REG_DWORD, 45) for name, value, type in self.test_data: SetValueEx(sub_key, name, 0, type, value) - exc = raises(TypeError, SetValueEx, sub_key, 'test_name', None, - REG_BINARY, memoryview('abc')) - assert str(exc.value) == ("Objects of type 'memoryview' can not " - "be used as binary registry values") + # cannot wrap a memoryview in setup_class for test_data + SetValueEx(sub_key, u'test_name', None, + REG_BINARY, memoryview(b'abc')) def test_readValues(self): from winreg import OpenKey, EnumValue, QueryValueEx, EnumKey @@ -163,26 +163,31 @@ data = EnumValue(sub_key, index) except EnvironmentError as e: break - assert data in self.test_data + if data[0] != 'test_name': + # cannot wrap a memoryview in setup_class for test_data + assert data in self.test_data index = index + 1 - assert index == len(self.test_data) + assert index == len(self.test_data) + 1 for name, value, type in self.test_data: result = QueryValueEx(sub_key, name) assert result == (value, type) if type == REG_SZ or type == REG_EXPAND_SZ: - assert isinstance(result[0], unicode) # not string + assert not isinstance(result[0], bytes) assert EnumKey(key, 0) == "sub_key" raises(EnvironmentError, EnumKey, key, 1) def test_delete(self): + # must be run after test_SetValueEx from winreg import OpenKey, KEY_ALL_ACCESS, DeleteValue, DeleteKey key = OpenKey(self.root_key, self.test_key_name, 0, KEY_ALL_ACCESS) sub_key = OpenKey(key, "sub_key", 0, KEY_ALL_ACCESS) for name, value, type in self.test_data: DeleteValue(sub_key, name) + # cannot wrap a memoryview in setup_class for test_data + DeleteValue(sub_key, 'test_name') DeleteKey(key, "sub_key") @@ -192,6 +197,7 @@ h.Close() def test_savekey(self): + # must be run after test_SetValueEx if not self.canSaveKey: skip("CPython needs win32api to set the SeBackupPrivilege security privilege") from winreg import OpenKey, KEY_ALL_ACCESS, SaveKey @@ -240,7 +246,7 @@ skip("access denied to registry key " "(are you running in a non-interactive session?)") raise - QueryValueEx(HKEY_PERFORMANCE_DATA, None) + QueryValueEx(HKEY_PERFORMANCE_DATA, 'Global') def test_reflection_unsupported(self): import sys diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py --- a/rpython/rlib/rdynload.py +++ b/rpython/rlib/rdynload.py @@ -228,7 +228,9 @@ res = rwin32.LoadLibrary(name) if not res: err = rwin32.GetLastError_saved() - raise DLOpenError(rwin32.FormatError(err)) + ustr = rwin32.FormatErrorW(err) + # DLOpenError unicode msg breaks translation of cpyext create_extension_module + raise DLOpenError(ustr.encode('utf-8')) return res def dlclose(handle): diff --git a/rpython/rlib/rwinreg.py b/rpython/rlib/rwinreg.py --- a/rpython/rlib/rwinreg.py +++ b/rpython/rlib/rwinreg.py @@ -47,71 +47,72 @@ HKEY = rwin32.HANDLE PHKEY = rffi.CArrayPtr(HKEY) REGSAM = rwin32.DWORD +suffix = 'W' RegSetValue = external( - 'RegSetValueA', + 'RegSetValue' + suffix, [HKEY, rffi.CCHARP, rwin32.DWORD, rffi.CCHARP, rwin32.DWORD], rffi.LONG) RegSetValueEx = external( - 'RegSetValueExA', + 'RegSetValueEx' + suffix, [HKEY, rffi.CCHARP, rwin32.DWORD, rwin32.DWORD, rffi.CCHARP, rwin32.DWORD], rffi.LONG) RegQueryValue = external( - 'RegQueryValueA', + 'RegQueryValue' + suffix, [HKEY, rffi.CCHARP, rffi.CCHARP, rwin32.PLONG], rffi.LONG) RegQueryValueEx = external( - 'RegQueryValueExA', + 'RegQueryValueEx' + suffix, [HKEY, rffi.CCHARP, rwin32.LPDWORD, rwin32.LPDWORD, rffi.CCHARP, rwin32.LPDWORD], rffi.LONG) RegCreateKey = external( - 'RegCreateKeyA', + 'RegCreateKey' + suffix, [HKEY, rffi.CCHARP, PHKEY], rffi.LONG) RegCreateKeyEx = external( - 'RegCreateKeyExA', + 'RegCreateKeyEx' + suffix, [HKEY, rffi.CCHARP, rwin32.DWORD, rffi.CCHARP, rwin32.DWORD, REGSAM, rffi.VOIDP, PHKEY, rwin32.LPDWORD], rffi.LONG) RegDeleteValue = external( - 'RegDeleteValueA', + 'RegDeleteValue' + suffix, [HKEY, rffi.CCHARP], rffi.LONG) RegDeleteKey = external( - 'RegDeleteKeyA', + 'RegDeleteKey' + suffix, [HKEY, rffi.CCHARP], rffi.LONG) RegOpenKeyEx = external( - 'RegOpenKeyExA', + 'RegOpenKeyEx' + suffix, [HKEY, rffi.CCHARP, rwin32.DWORD, REGSAM, PHKEY], rffi.LONG) RegEnumValue = external( - 'RegEnumValueA', + 'RegEnumValue' + suffix, [HKEY, rwin32.DWORD, rffi.CCHARP, rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, rffi.CCHARP, rwin32.LPDWORD], rffi.LONG) RegEnumKeyEx = external( - 'RegEnumKeyExA', + 'RegEnumKeyEx' + suffix, [HKEY, rwin32.DWORD, rffi.CCHARP, rwin32.LPDWORD, rwin32.LPDWORD, rffi.CCHARP, rwin32.LPDWORD, rwin32.PFILETIME], rffi.LONG) RegQueryInfoKey = external( - 'RegQueryInfoKeyA', + 'RegQueryInfoKey' + suffix, [HKEY, rffi.CCHARP, rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, @@ -129,17 +130,17 @@ rffi.LONG) RegLoadKey = external( - 'RegLoadKeyA', + 'RegLoadKey' + suffix, [HKEY, rffi.CCHARP, rffi.CCHARP], rffi.LONG) RegSaveKey = external( - 'RegSaveKeyA', + 'RegSaveKey' + suffix, [HKEY, rffi.CCHARP, rffi.VOIDP], rffi.LONG) RegConnectRegistry = external( - 'RegConnectRegistryA', + 'RegConnectRegistry' + suffix, [rffi.CCHARP, HKEY, PHKEY], rffi.LONG) diff --git a/testrunner/get_info.py b/testrunner/get_info.py --- a/testrunner/get_info.py +++ b/testrunner/get_info.py @@ -8,14 +8,21 @@ import json BASE_DIR = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) -TARGET_BASENAME = 'pypy3-c' +if sys.platform.startswith('win'): + TARGET_NAME = r'pypy3-c.exe' + # see https://github.com/pypa/virtualenv/issues/789 + TARGET_DIR = 'bin' +else: + TARGET_NAME = 'pypy3-c' + TARGET_DIR = 'bin' +VENV_DIR = 'pypy-venv' def make_info_dict(): - target = TARGET_BASENAME - if sys.platform.startswith('win'): - target += '.exe' - target_path = os.path.join(BASE_DIR, 'pypy', 'goal', target) - return {'target_path': target_path} + target_path = os.path.join(BASE_DIR, 'pypy', 'goal', TARGET_NAME) + return {'target_path': target_path, + 'virt_pypy': os.path.join(VENV_DIR, TARGET_DIR, TARGET_NAME), + 'venv_dir': VENV_DIR, + } def dump_info(): return json.dumps(make_info_dict()) From pypy.commits at gmail.com Mon Jan 8 13:46:47 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 08 Jan 2018 10:46:47 -0800 (PST) Subject: [pypy-commit] pypy default: start 5.10.1 release notes Message-ID: <5a53bc97.54badf0a.cde0b.a382@mx.google.com> Author: Matti Picus Branch: Changeset: r93640:1c81e9755c00 Date: 2018-01-08 20:18 +0200 http://bitbucket.org/pypy/pypy/changeset/1c81e9755c00/ Log: start 5.10.1 release notes diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-v5.10.1.rst release-v5.10.0.rst release-v5.9.0.rst release-v5.8.0.rst diff --git a/pypy/doc/release-v5.10.1.rst b/pypy/doc/release-v5.10.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v5.10.1.rst @@ -0,0 +1,63 @@ +=========== +PyPy 5.10.1 +=========== + +We have released a bugfix PyPy3.5-v5.10.1 +due to the following issues: + + * Fix ``time.sleep(float('nan')`` which would hang on windows + + * Fix missing ``errno`` constants on windows + + * Fix issue 2718_ for the REPL on linux + + * Fix an overflow in converting 3 secs to nanosecs (issue 2717_ ) + + * Flag kwarg to ``os.setxattr`` had no effect + + * Fix the winreg module for unicode entries in the registry on windows + +Note that many of these fixes are for our new beta verison of PyPy3.5 on +windows. There may be more unicode problems in the windows beta version +especially around the subject of directory- and file-names with non-ascii +characters. + +Our downloads are available now. On macos, we recommend you wait for the +Homebrew_ package. + +Thanks to those who reported the issues. + +.. _2718: https://bitbucket.org/pypy/pypy/issues/2718 +.. _2717: https://bitbucket.org/pypy/pypy/issues/2717 +.. _Homebrew: http://brewformulas.org/Pypy + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +This PyPy 3.5 release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + +Please update, and continue to help us make PyPy better. + +Cheers + +The PyPy Team + From pypy.commits at gmail.com Mon Jan 8 16:09:24 2018 From: pypy.commits at gmail.com (arigo) Date: Mon, 08 Jan 2018 13:09:24 -0800 (PST) Subject: [pypy-commit] cffi default: Issue #351 Message-ID: <5a53de04.6286df0a.5e3f8.8e93@mx.google.com> Author: Armin Rigo Branch: Changeset: r3057:dfb27809ae0f Date: 2018-01-08 22:09 +0100 http://bitbucket.org/cffi/cffi/changeset/dfb27809ae0f/ Log: Issue #351 improve error message in this deprecated file diff --git a/cffi/verifier.py b/cffi/verifier.py --- a/cffi/verifier.py +++ b/cffi/verifier.py @@ -301,7 +301,6 @@ return suffixes def _ensure_dir(filename): - try: - os.makedirs(os.path.dirname(filename)) - except OSError: - pass + dirname = os.path.dirname(filename) + if dirname and not os.path.isdir(dirname): + os.makedirs(dirname) From pypy.commits at gmail.com Mon Jan 8 16:20:22 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 08 Jan 2018 13:20:22 -0800 (PST) Subject: [pypy-commit] pypy py3.5: fix/skip easy tests (test_codecs checked on cpython3) Message-ID: <5a53e096.79b8df0a.3c464.2430@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93641:f5c9251c71a6 Date: 2018-01-08 23:19 +0200 http://bitbucket.org/pypy/pypy/changeset/f5c9251c71a6/ Log: fix/skip easy tests (test_codecs checked on cpython3) diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -892,7 +892,7 @@ assert False, 'cannot test mbcs on this windows system, check code page' assert u'test'.encode('mbcs') == b'test' assert toencode[0].encode('mbcs') == toencode[1] - assert u'\u040a'.encode('mbcs') == b'?' # some cyrillic letter + raises(UnicodeEncodeError, u'\u040a'.encode, 'mbcs') assert b'cafx\e9'.decode('mbcs') == u'cafx\e9' def test_handler_string_result(self): diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -1424,6 +1424,7 @@ skip("_getfinalpathname not supported on this platform") assert os.path.exists(result) + @py.test.mark.skipif("sys.platform == 'win32'") def test_rtld_constants(self): # check presence of major RTLD_* constants self.posix.RTLD_LAZY @@ -1432,6 +1433,7 @@ self.posix.RTLD_LOCAL def test_error_message(self): + import sys e = raises(OSError, self.posix.open, 'nonexistentfile1', 0) assert str(e.value).endswith(": 'nonexistentfile1'") @@ -1442,8 +1444,9 @@ e = raises(OSError, self.posix.replace, 'nonexistentfile1', 'bok') assert str(e.value).endswith(": 'nonexistentfile1' -> 'bok'") - e = raises(OSError, self.posix.symlink, 'bok', '/nonexistentdir/boz') - assert str(e.value).endswith(": 'bok' -> '/nonexistentdir/boz'") + if sys.platform != 'win32': + e = raises(OSError, self.posix.symlink, 'bok', '/nonexistentdir/boz') + assert str(e.value).endswith(": 'bok' -> '/nonexistentdir/boz'") if hasattr(rposix, 'getxattr'): def test_xattr_simple(self): @@ -1472,8 +1475,8 @@ cls.w_path = space.wrap(str(path)) def test_environ(self): - import sys, posix - environ = posix.environ + import sys, os + environ = os.environ item_type = str if sys.platform.startswith('win') else bytes for k, v in environ.items(): assert type(k) is item_type @@ -1536,25 +1539,25 @@ class AppTestPosixUnicode: def test_stat_unicode(self): # test that passing unicode would not raise UnicodeDecodeError - import posix + import os try: - posix.stat(u"ą") + os.stat(u"ą") except OSError: pass def test_open_unicode(self): # Ensure passing unicode doesn't raise UnicodeEncodeError - import posix + import os try: - posix.open(u"ą", posix.O_WRONLY) + os.open(u"ą", os.O_WRONLY) except OSError: pass def test_remove_unicode(self): # See 2 above ;) - import posix + import os try: - posix.remove(u"ą") + os.remove(u"ą") except OSError: pass From pypy.commits at gmail.com Tue Jan 9 12:56:56 2018 From: pypy.commits at gmail.com (mjacob) Date: Tue, 09 Jan 2018 09:56:56 -0800 (PST) Subject: [pypy-commit] extradoc extradoc: Add myself. Message-ID: <5a550268.e791df0a.17e33.ab92@mx.google.com> Author: Manuel Jacob Branch: extradoc Changeset: r5862:66b597d8bf63 Date: 2018-01-09 18:45 +0100 http://bitbucket.org/pypy/extradoc/changeset/66b597d8bf63/ Log: Add myself. diff --git a/sprintinfo/leysin-winter-2018/people.txt b/sprintinfo/leysin-winter-2018/people.txt --- a/sprintinfo/leysin-winter-2018/people.txt +++ b/sprintinfo/leysin-winter-2018/people.txt @@ -11,6 +11,7 @@ ==================== ============== ======================= Armin Rigo private Matti Picus 17.3/22.3 Ermina +Manuel Jacob 17.3/24.3 Ermina ==================== ============== ======================= **NOTE:** lodging is by default in Ermina. Based on past years, there From pypy.commits at gmail.com Tue Jan 9 17:45:29 2018 From: pypy.commits at gmail.com (arigo) Date: Tue, 09 Jan 2018 14:45:29 -0800 (PST) Subject: [pypy-commit] pypy default: Print some more stats at the end of a major GC Message-ID: <5a554609.d3c7df0a.5df1d.c52b@mx.google.com> Author: Armin Rigo Branch: Changeset: r93642:e21bb2a19d05 Date: 2018-01-09 23:44 +0100 http://bitbucket.org/pypy/pypy/changeset/e21bb2a19d05/ Log: Print some more stats at the end of a major GC diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -2354,6 +2354,8 @@ if self.rrc_enabled: self.rrc_major_collection_free() # + self.stat_ac_arenas_count = self.ac.arenas_count + self.stat_rawmalloced_total_size = self.rawmalloced_total_size self.gc_state = STATE_SWEEPING #END MARKING elif self.gc_state == STATE_SWEEPING: @@ -2383,6 +2385,18 @@ # We also need to reset the GCFLAG_VISITED on prebuilt GC objects. self.prebuilt_root_objects.foreach(self._reset_gcflag_visited, None) # + # Print statistics + debug_start("gc-collect-done") + debug_print("arenas: ", + self.stat_ac_arenas_count, " => ", + self.ac.arenas_count) + debug_print("bytes used in arenas: ", + self.ac.total_memory_used) + debug_print("bytes raw-malloced: ", + self.stat_rawmalloced_total_size, " => ", + self.rawmalloced_total_size) + debug_stop("gc-collect-done") + # # Set the threshold for the next major collection to be when we # have allocated 'major_collection_threshold' times more than # we currently have -- but no more than 'max_delta' more than diff --git a/rpython/memory/gc/minimarkpage.py b/rpython/memory/gc/minimarkpage.py --- a/rpython/memory/gc/minimarkpage.py +++ b/rpython/memory/gc/minimarkpage.py @@ -94,6 +94,7 @@ self.arena_size = arena_size self.page_size = page_size self.small_request_threshold = small_request_threshold + self.arenas_count = 0 # # 'pageaddr_for_size': for each size N between WORD and # small_request_threshold (included), contains either NULL or @@ -311,6 +312,7 @@ arena.freepages = firstpage self.num_uninitialized_pages = npages self.current_arena = arena + self.arenas_count += 1 # allocate_new_arena._dont_inline_ = True @@ -398,6 +400,7 @@ llarena.arena_reset(arena.base, self.arena_size, 4) llarena.arena_free(arena.base) lltype.free(arena, flavor='raw', track_allocation=False) + self.arenas_count -= 1 # else: # Insert 'arena' in the correct arenas_lists[n] From pypy.commits at gmail.com Wed Jan 10 07:03:35 2018 From: pypy.commits at gmail.com (arigo) Date: Wed, 10 Jan 2018 04:03:35 -0800 (PST) Subject: [pypy-commit] cffi doc-set_source: Close branch Message-ID: <5a560117.88b0df0a.8298.0609@mx.google.com> Author: Armin Rigo Branch: doc-set_source Changeset: r3058:237749810e6e Date: 2018-01-10 12:50 +0100 http://bitbucket.org/cffi/cffi/changeset/237749810e6e/ Log: Close branch From pypy.commits at gmail.com Wed Jan 10 07:03:37 2018 From: pypy.commits at gmail.com (arigo) Date: Wed, 10 Jan 2018 04:03:37 -0800 (PST) Subject: [pypy-commit] cffi default: hg merge doc-set_source Message-ID: <5a560119.f8b8df0a.7a542.f213@mx.google.com> Author: Armin Rigo Branch: Changeset: r3059:a570b34ac8a6 Date: 2018-01-10 12:51 +0100 http://bitbucket.org/cffi/cffi/changeset/a570b34ac8a6/ Log: hg merge doc-set_source Thanks Matti! diff --git a/doc/source/overview.rst b/doc/source/overview.rst --- a/doc/source/overview.rst +++ b/doc/source/overview.rst @@ -45,8 +45,9 @@ there, %s!\n"``. In general it is ``somestring.encode(myencoding)``. *Python 3 on Windows:* ``ffi.dlopen(None)`` does not work. This problem -is messy and not really fixable. The example above could be fixed by -calling another function from a specific DLL that exists on your system. +is messy and not really fixable. The problem does not occur if you try +to call a fucntion from a specific DLL that exists on your system: then +you use ``ffi.dlopen("path.dll")``. *This example does not call any C compiler. It works in the so-called ABI mode, which means that it will crash if you call some function or @@ -76,20 +77,27 @@ ffibuilder = FFI() ffibuilder.set_source("_example", - r""" // passed to the real C compiler + r""" // passed to the real C compiler, + // contains implementation of things declared in cdef() #include #include + + struct passwd *get_pw_for_root(void) { + return getpwuid(0); + } """, libraries=[]) # or a list of libraries to link with # (more arguments like setup.py's Extension class: # include_dirs=[..], extra_objects=[..], and so on) - ffibuilder.cdef(""" // some declarations from the man page + ffibuilder.cdef(""" + // declarations that are shared between Python and C struct passwd { char *pw_name; ...; // literally dot-dot-dot }; - struct passwd *getpwuid(int uid); + struct passwd *getpwuid(int uid); // defined in + struct passwd *get_pw_for_root(void); // defined in set_source() """) if __name__ == "__main__": @@ -113,15 +121,18 @@ p = lib.getpwuid(0) assert ffi.string(p.pw_name) == b'root' + p = lib.get_pw_for_root() + assert ffi.string(p.pw_name) == b'root' Note that this works independently of the exact C layout of ``struct passwd`` (it is "API level", as opposed to "ABI level"). It requires a C compiler in order to run ``example_build.py``, but it is much more portable than trying to get the details of the fields of ``struct -passwd`` exactly right. Similarly, we declared ``getpwuid()`` as -taking an ``int`` argument. On some platforms this might be slightly -incorrect---but it does not matter. It is also faster than the ABI -mode. +passwd`` exactly right. Similarly, in the ``cdef()`` we declared +``getpwuid()`` as taking an ``int`` argument; on some platforms this +might be slightly incorrect---but it does not matter. + +Note also that at runtime, the API mode is faster than the ABI mode. To integrate it inside a ``setup.py`` distribution with Setuptools: From pypy.commits at gmail.com Wed Jan 10 07:05:22 2018 From: pypy.commits at gmail.com (arigo) Date: Wed, 10 Jan 2018 04:05:22 -0800 (PST) Subject: [pypy-commit] pypy default: Fix following 0d3c3f5e2bdb Message-ID: <5a560182.c2b2df0a.6278.8361@mx.google.com> Author: Armin Rigo Branch: Changeset: r93643:1e390e19899b Date: 2018-01-10 12:54 +0100 http://bitbucket.org/pypy/pypy/changeset/1e390e19899b/ Log: Fix following 0d3c3f5e2bdb diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py --- a/pypy/module/pypyjit/test_pypy_c/test_string.py +++ b/pypy/module/pypyjit/test_pypy_c/test_string.py @@ -76,7 +76,7 @@ i23 = strgetitem(p10, i19) p25 = newstr(1) strsetitem(p25, 0, i23) - p93 = call_r(ConstClass(fromstr), p25, 16, descr=) + p93 = call_r(ConstClass(fromstr), p25, 16, 0, descr=) guard_no_exception(descr=...) i95 = getfield_gc_i(p93, descr=) i96 = int_gt(i95, #) From pypy.commits at gmail.com Wed Jan 10 11:09:42 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 10 Jan 2018 08:09:42 -0800 (PST) Subject: [pypy-commit] pypy release-pypy3.5-v5.9.x: missed this skip in the merge Message-ID: <5a563ac6.89a3df0a.ce316.29ad@mx.google.com> Author: Matti Picus Branch: release-pypy3.5-v5.9.x Changeset: r93644:3f6eaa010fce Date: 2018-01-10 16:58 +0200 http://bitbucket.org/pypy/pypy/changeset/3f6eaa010fce/ Log: missed this skip in the merge diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py --- a/pypy/module/cpyext/test/test_eval.py +++ b/pypy/module/cpyext/test/test_eval.py @@ -344,6 +344,7 @@ return module.get_flags()""", ns) assert ns['nested_flags']() == (0, 0) + @pytest.mark.xfail("sys.platform == 'win32'", reason='Hangs the process', run=False) def test_recursive_function(self): module = self.import_extension('foo', [ ("call_recursive", "METH_NOARGS", From pypy.commits at gmail.com Wed Jan 10 11:09:44 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 10 Jan 2018 08:09:44 -0800 (PST) Subject: [pypy-commit] pypy default: update year Message-ID: <5a563ac8.12711c0a.79577.e05e@mx.google.com> Author: Matti Picus Branch: Changeset: r93645:656a15610ee8 Date: 2018-01-10 17:01 +0200 http://bitbucket.org/pypy/pypy/changeset/656a15610ee8/ Log: update year diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -30,7 +30,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2017 +PyPy Copyright holders 2003-2018 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -59,7 +59,7 @@ # General information about the project. project = u'PyPy' -copyright = u'2017, The PyPy Project' +copyright = u'2018, The PyPy Project' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From pypy.commits at gmail.com Wed Jan 10 11:09:46 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 10 Jan 2018 08:09:46 -0800 (PST) Subject: [pypy-commit] pypy default: trivial fixes Message-ID: <5a563aca.0ba3df0a.4fe9e.f37f@mx.google.com> Author: Matti Picus Branch: Changeset: r93646:35cff1db08c1 Date: 2018-01-10 18:08 +0200 http://bitbucket.org/pypy/pypy/changeset/35cff1db08c1/ Log: trivial fixes diff --git a/rpython/memory/gc/minimarktest.py b/rpython/memory/gc/minimarktest.py --- a/rpython/memory/gc/minimarktest.py +++ b/rpython/memory/gc/minimarktest.py @@ -20,6 +20,7 @@ self.small_request_threshold = small_request_threshold self.all_objects = [] self.total_memory_used = 0 + self.arenas_count = 0 def malloc(self, size): nsize = raw_malloc_usage(size) diff --git a/rpython/rlib/test/test_rarithmetic.py b/rpython/rlib/test/test_rarithmetic.py --- a/rpython/rlib/test/test_rarithmetic.py +++ b/rpython/rlib/test/test_rarithmetic.py @@ -559,7 +559,7 @@ '4_2', '1_0000_0000', '0b1001_0100', - '0xffff_ffff', + '0xfff_ffff', '0o5_7_7', '0b_0', '0x_f', @@ -583,7 +583,7 @@ # Multiple consecutive underscores: '4_______2', '0b1001__0100', - '0xffff__ffff', + '0xfff__ffff', '0x___', '0o5__77', '1e1__0', From pypy.commits at gmail.com Wed Jan 10 15:35:44 2018 From: pypy.commits at gmail.com (rlamy) Date: Wed, 10 Jan 2018 12:35:44 -0800 (PST) Subject: [pypy-commit] pypy default: Backport rpython/ changes made on the py3.5 branch Message-ID: <5a567920.03da1c0a.40caa.7767@mx.google.com> Author: Ronan Lamy Branch: Changeset: r93648:f560bd567d72 Date: 2018-01-10 20:34 +0000 http://bitbucket.org/pypy/pypy/changeset/f560bd567d72/ Log: Backport rpython/ changes made on the py3.5 branch diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py --- a/rpython/annotator/binaryop.py +++ b/rpython/annotator/binaryop.py @@ -381,16 +381,14 @@ class __extend__(pairtype(SomeChar, SomeUnicodeCodePoint), pairtype(SomeUnicodeCodePoint, SomeChar)): def union((uchr1, uchr2)): - return SomeUnicodeCodePoint() + no_nul = uchr1.no_nul and uchr2.no_nul + return SomeUnicodeCodePoint(no_nul=no_nul) class __extend__(pairtype(SomeUnicodeCodePoint, SomeUnicodeCodePoint)): def union((uchr1, uchr2)): no_nul = uchr1.no_nul and uchr2.no_nul return SomeUnicodeCodePoint(no_nul=no_nul) - def add((chr1, chr2)): - return SomeUnicodeString() - class __extend__(pairtype(SomeString, SomeUnicodeString), pairtype(SomeUnicodeString, SomeString)): def mod((str, unistring)): diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py --- a/rpython/rlib/rdynload.py +++ b/rpython/rlib/rdynload.py @@ -228,7 +228,9 @@ res = rwin32.LoadLibrary(name) if not res: err = rwin32.GetLastError_saved() - raise DLOpenError(rwin32.FormatError(err)) + ustr = rwin32.FormatErrorW(err) + # DLOpenError unicode msg breaks translation of cpyext create_extension_module + raise DLOpenError(ustr.encode('utf-8')) return res def dlclose(handle): diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -2574,3 +2574,160 @@ """Passes offset==NULL; not support on all OSes""" res = c_sendfile(out_fd, in_fd, lltype.nullptr(_OFF_PTR_T.TO), count) return handle_posix_error('sendfile', res) + +# ____________________________________________________________ +# Support for *xattr functions + +if sys.platform.startswith('linux'): + + class CConfig: + _compilation_info_ = ExternalCompilationInfo( + includes=['sys/xattr.h', 'linux/limits.h'],) + XATTR_SIZE_MAX = rffi_platform.DefinedConstantInteger('XATTR_SIZE_MAX') + XATTR_CREATE = rffi_platform.DefinedConstantInteger('XATTR_CREATE') + XATTR_REPLACE = rffi_platform.DefinedConstantInteger('XATTR_REPLACE') + + cConfig = rffi_platform.configure(CConfig) + globals().update(cConfig) + c_fgetxattr = external('fgetxattr', + [rffi.INT, rffi.CCHARP, rffi.VOIDP, rffi.SIZE_T], rffi.SSIZE_T, + compilation_info=CConfig._compilation_info_, + save_err=rffi.RFFI_SAVE_ERRNO) + c_getxattr = external('getxattr', + [rffi.CCHARP, rffi.CCHARP, rffi.VOIDP, rffi.SIZE_T], rffi.SSIZE_T, + compilation_info=CConfig._compilation_info_, + save_err=rffi.RFFI_SAVE_ERRNO) + c_lgetxattr = external('lgetxattr', + [rffi.CCHARP, rffi.CCHARP, rffi.VOIDP, rffi.SIZE_T], rffi.SSIZE_T, + compilation_info=CConfig._compilation_info_, + save_err=rffi.RFFI_SAVE_ERRNO) + c_fsetxattr = external('fsetxattr', + [rffi.INT, rffi.CCHARP, rffi.CCHARP, rffi.SIZE_T, rffi.INT], + rffi.INT, + compilation_info=CConfig._compilation_info_, + save_err=rffi.RFFI_SAVE_ERRNO) + c_setxattr = external('setxattr', + [rffi.CCHARP, rffi.CCHARP, rffi.CCHARP, rffi.SIZE_T, rffi.INT], + rffi.INT, + compilation_info=CConfig._compilation_info_, + save_err=rffi.RFFI_SAVE_ERRNO) + c_lsetxattr = external('lsetxattr', + [rffi.CCHARP, rffi.CCHARP, rffi.CCHARP, rffi.SIZE_T, rffi.INT], + rffi.INT, + compilation_info=CConfig._compilation_info_, + save_err=rffi.RFFI_SAVE_ERRNO) + c_fremovexattr = external('fremovexattr', + [rffi.INT, rffi.CCHARP], rffi.INT, + compilation_info=CConfig._compilation_info_, + save_err=rffi.RFFI_SAVE_ERRNO) + c_removexattr = external('removexattr', + [rffi.CCHARP, rffi.CCHARP], rffi.INT, + compilation_info=CConfig._compilation_info_, + save_err=rffi.RFFI_SAVE_ERRNO) + c_lremovexattr = external('lremovexattr', + [rffi.CCHARP, rffi.CCHARP], rffi.INT, + compilation_info=CConfig._compilation_info_, + save_err=rffi.RFFI_SAVE_ERRNO) + c_flistxattr = external('flistxattr', + [rffi.INT, rffi.CCHARP, rffi.SIZE_T], rffi.SSIZE_T, + compilation_info=CConfig._compilation_info_, + save_err=rffi.RFFI_SAVE_ERRNO) + c_listxattr = external('listxattr', + [rffi.CCHARP, rffi.CCHARP, rffi.SIZE_T], rffi.SSIZE_T, + compilation_info=CConfig._compilation_info_, + save_err=rffi.RFFI_SAVE_ERRNO) + c_llistxattr = external('llistxattr', + [rffi.CCHARP, rffi.CCHARP, rffi.SIZE_T], rffi.SSIZE_T, + compilation_info=CConfig._compilation_info_, + save_err=rffi.RFFI_SAVE_ERRNO) + buf_sizes = [256, XATTR_SIZE_MAX] + + def fgetxattr(fd, name): + for size in buf_sizes: + with rffi.scoped_alloc_buffer(size) as buf: + void_buf = rffi.cast(rffi.VOIDP, buf.raw) + res = c_fgetxattr(fd, name, void_buf, size) + if res < 0: + err = get_saved_errno() + if err != errno.ERANGE: + raise OSError(err, 'fgetxattr failed') + else: + return buf.str(res) + else: + raise OSError(errno.ERANGE, 'fgetxattr failed') + + def getxattr(path, name, follow_symlinks=True): + for size in buf_sizes: + with rffi.scoped_alloc_buffer(size) as buf: + void_buf = rffi.cast(rffi.VOIDP, buf.raw) + if follow_symlinks: + res = c_getxattr(path, name, void_buf, size) + else: + res = c_lgetxattr(path, name, void_buf, size) + if res < 0: + err = get_saved_errno() + if err != errno.ERANGE: + c_name = 'getxattr' if follow_symlinks else 'lgetxattr' + raise OSError(err, c_name + 'failed') + else: + return buf.str(res) + else: + c_name = 'getxattr' if follow_symlinks else 'lgetxattr' + raise OSError(errno.ERANGE, c_name + 'failed') + + def fsetxattr(fd, name, value, flags=0): + return handle_posix_error( + 'fsetxattr', c_fsetxattr(fd, name, value, len(value), flags)) + + def setxattr(path, name, value, flags=0, follow_symlinks=True): + if follow_symlinks: + return handle_posix_error( + 'setxattr', c_setxattr(path, name, value, len(value), flags)) + else: + return handle_posix_error( + 'lsetxattr', c_lsetxattr(path, name, value, len(value), flags)) + + def fremovexattr(fd, name): + return handle_posix_error('fremovexattr', c_fremovexattr(fd, name)) + + def removexattr(path, name, follow_symlinks=True): + if follow_symlinks: + return handle_posix_error('removexattr', c_removexattr(path, name)) + else: + return handle_posix_error('lremovexattr', c_lremovexattr(path, name)) + + def _unpack_attrs(attr_string): + result = attr_string.split('\0') + del result[-1] + return result + + def flistxattr(fd): + for size in buf_sizes: + with rffi.scoped_alloc_buffer(size) as buf: + res = c_flistxattr(fd, buf.raw, size) + if res < 0: + err = get_saved_errno() + if err != errno.ERANGE: + raise OSError(err, 'flistxattr failed') + else: + return _unpack_attrs(buf.str(res)) + else: + raise OSError(errno.ERANGE, 'flistxattr failed') + + def listxattr(path, follow_symlinks=True): + for size in buf_sizes: + with rffi.scoped_alloc_buffer(size) as buf: + if follow_symlinks: + res = c_listxattr(path, buf.raw, size) + else: + res = c_llistxattr(path, buf.raw, size) + if res < 0: + err = get_saved_errno() + if err != errno.ERANGE: + c_name = 'listxattr' if follow_symlinks else 'llistxattr' + raise OSError(err, c_name + 'failed') + else: + return _unpack_attrs(buf.str(res)) + else: + c_name = 'listxattr' if follow_symlinks else 'llistxattr' + raise OSError(errno.ERANGE, c_name + 'failed') diff --git a/rpython/rlib/rposix_scandir.py b/rpython/rlib/rposix_scandir.py --- a/rpython/rlib/rposix_scandir.py +++ b/rpython/rlib/rposix_scandir.py @@ -1,56 +1,126 @@ from rpython.rlib import rposix, rwin32 from rpython.rlib.objectmodel import specialize from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib.rarithmetic import intmask - at specialize.argtype(0) -def opendir(path): - path = rposix._as_bytes0(path) - return opendir_bytes(path) - -def opendir_bytes(path): - dirp = rposix.c_opendir(path) - if not dirp: - raise OSError(rposix.get_saved_errno(), "opendir failed") - return dirp - -def closedir(dirp): - rposix.c_closedir(dirp) - if not rwin32.WIN32: + @specialize.argtype(0) + def opendir(path): + path = rposix._as_bytes0(path) + return opendir_bytes(path) + + def opendir_bytes(path): + dirp = rposix.c_opendir(path) + if not dirp: + raise OSError(rposix.get_saved_errno(), "opendir failed") + return dirp + + def closedir(dirp): + rposix.c_closedir(dirp) + NULL_DIRP = lltype.nullptr(rposix.DIRP.TO) -def nextentry(dirp): - """Read the next entry and returns an opaque object. - Use the methods has_xxx() and get_xxx() to read from that - opaque object. The opaque object is valid until the next - time nextentry() or closedir() is called. This may raise - OSError, or return a NULL pointer when exhausted. Note - that this doesn't filter out the "." and ".." entries. - """ - direntp = rposix.c_readdir(dirp) - if direntp: - error = rposix.get_saved_errno() - if error: - raise OSError(error, "readdir failed") - return direntp + def nextentry(dirp): + """Read the next entry and returns an opaque object. + Use the methods has_xxx() and get_xxx() to read from that + opaque object. The opaque object is valid until the next + time nextentry() or closedir() is called. This may raise + OSError, or return a NULL pointer when exhausted. Note + that this doesn't filter out the "." and ".." entries. + """ + direntp = rposix.c_readdir(dirp) + if direntp: + error = rposix.get_saved_errno() + if error: + raise OSError(error, "readdir failed") + return direntp -def has_name_bytes(direntp): - return True + def get_name_bytes(direntp): + namep = rffi.cast(rffi.CCHARP, direntp.c_d_name) + return rffi.charp2str(namep) -def get_name_bytes(direntp): - namep = rffi.cast(rffi.CCHARP, direntp.c_d_name) - return rffi.charp2str(namep) + DT_UNKNOWN = rposix.dirent_config.get('DT_UNKNOWN', 0) + DT_REG = rposix.dirent_config.get('DT_REG', 255) + DT_DIR = rposix.dirent_config.get('DT_DIR', 255) + DT_LNK = rposix.dirent_config.get('DT_LNK', 255) -DT_UNKNOWN = rposix.dirent_config.get('DT_UNKNOWN', 0) -DT_REG = rposix.dirent_config.get('DT_REG', 255) -DT_DIR = rposix.dirent_config.get('DT_DIR', 255) -DT_LNK = rposix.dirent_config.get('DT_LNK', 255) + def get_known_type(direntp): + if rposix.HAVE_D_TYPE: + return rffi.getintfield(direntp, 'c_d_type') + return DT_UNKNOWN -def get_known_type(direntp): - if rposix.HAVE_D_TYPE: - return rffi.getintfield(direntp, 'c_d_type') - return DT_UNKNOWN + def get_inode(direntp): + return rffi.getintfield(direntp, 'c_d_ino') -def get_inode(direntp): - return rffi.getintfield(direntp, 'c_d_ino') +else: + # ----- Win32 version ----- + import stat + from rpython.rlib._os_support import unicode_traits + from rpython.rlib.rwin32file import make_win32_traits + from rpython.rlib import rposix_stat + + win32traits = make_win32_traits(unicode_traits) + + + SCANDIRP = lltype.Ptr(lltype.Struct('SCANDIRP', + ('filedata', win32traits.WIN32_FIND_DATA), + ('hFindFile', rwin32.HANDLE), + ('first_time', lltype.Bool), + )) + NULL_DIRP = lltype.nullptr(SCANDIRP.TO) + + + # must only be called with unicode! + def opendir(path): + if len(path) == 0: + path = u'.' + if path[-1] not in (u'\\', u'/', u':'): + mask = path + u'\\*.*' + else: + mask = path + u'*.*' + dirp = lltype.malloc(SCANDIRP.TO, flavor='raw') + hFindFile = win32traits.FindFirstFile(mask, dirp.filedata) + if hFindFile == rwin32.INVALID_HANDLE_VALUE: + error = rwin32.GetLastError_saved() + lltype.free(dirp, flavor='raw') + raise WindowsError(error, "FindFirstFileW failed") + dirp.hFindFile = hFindFile + dirp.first_time = True + return dirp + + def closedir(dirp): + if dirp.hFindFile != rwin32.INVALID_HANDLE_VALUE: + win32traits.FindClose(dirp.hFindFile) + lltype.free(dirp, flavor='raw') + + def nextentry(dirp): + """Read the next entry and returns an opaque object. + Use the methods has_xxx() and get_xxx() to read from that + opaque object. The opaque object is valid until the next + time nextentry() or closedir() is called. This may raise + WindowsError, or return NULL when exhausted. Note + that this doesn't filter out the "." and ".." entries. + """ + if dirp.first_time: + dirp.first_time = False + else: + if not win32traits.FindNextFile(dirp.hFindFile, dirp.filedata): + # error or no more files + error = rwin32.GetLastError_saved() + if error == win32traits.ERROR_NO_MORE_FILES: + return lltype.nullptr(win32traits.WIN32_FIND_DATA) + raise WindowsError(error, "FindNextFileW failed") + return dirp.filedata + + def get_name_unicode(filedata): + return unicode_traits.charp2str(rffi.cast(unicode_traits.CCHARP, + filedata.c_cFileName)) + + def get_known_type(filedata): + attr = filedata.c_dwFileAttributes + st_mode = rposix_stat.win32_attributes_to_mode(win32traits, attr) + return stat.S_IFMT(st_mode) + + def get_inode(filedata): + return None diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py --- a/rpython/rlib/rtime.py +++ b/rpython/rlib/rtime.py @@ -165,7 +165,7 @@ 'QueryPerformanceCounter', [rffi.CArrayPtr(lltype.SignedLongLong)], lltype.Void, releasegil=False) QueryPerformanceFrequency = external( - 'QueryPerformanceFrequency', [rffi.CArrayPtr(lltype.SignedLongLong)], + 'QueryPerformanceFrequency', [rffi.CArrayPtr(lltype.SignedLongLong)], rffi.INT, releasegil=False) class State(object): divisor = 0.0 @@ -267,9 +267,10 @@ else: void = lltype.nullptr(rffi.VOIDP.TO) with lltype.scoped_alloc(TIMEVAL) as t: - frac = math.fmod(secs, 1.0) + frac = int(math.fmod(secs, 1.0) * 1000000.) + assert frac >= 0 rffi.setintfield(t, 'c_tv_sec', int(secs)) - rffi.setintfield(t, 'c_tv_usec', int(frac*1000000.0)) + rffi.setintfield(t, 'c_tv_usec', frac) if rffi.cast(rffi.LONG, c_select(0, void, void, void, t)) != 0: errno = rposix.get_saved_errno() diff --git a/rpython/rlib/rwin32file.py b/rpython/rlib/rwin32file.py --- a/rpython/rlib/rwin32file.py +++ b/rpython/rlib/rwin32file.py @@ -148,7 +148,7 @@ save_err=rffi.RFFI_SAVE_LASTERROR) FindClose = external('FindClose', [rwin32.HANDLE], - rwin32.BOOL) + rwin32.BOOL, releasegil=False) GetFileAttributes = external( 'GetFileAttributes' + suffix, diff --git a/rpython/rlib/rwinreg.py b/rpython/rlib/rwinreg.py --- a/rpython/rlib/rwinreg.py +++ b/rpython/rlib/rwinreg.py @@ -47,7 +47,7 @@ HKEY = rwin32.HANDLE PHKEY = rffi.CArrayPtr(HKEY) REGSAM = rwin32.DWORD -suffix = 'A' +suffix = 'W' RegSetValue = external( 'RegSetValue' + suffix, diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py --- a/rpython/rlib/test/test_rposix.py +++ b/rpython/rlib/test/test_rposix.py @@ -1,3 +1,6 @@ +from hypothesis import given, strategies as st, assume +import pytest + from rpython.rtyper.test.test_llinterp import interpret from rpython.translator.c.test.test_genc import compile from rpython.tool.pytest.expecttest import ExpectTest @@ -8,10 +11,10 @@ import py def rposix_requires(funcname): - return py.test.mark.skipif(not hasattr(rposix, funcname), + return pytest.mark.skipif(not hasattr(rposix, funcname), reason="Requires rposix.%s()" % funcname) -win_only = py.test.mark.skipif("os.name != 'nt'") +win_only = pytest.mark.skipif("os.name != 'nt'") class TestPosixFunction: def test_access(self): @@ -827,3 +830,61 @@ rposix.lockf(fd, rposix.F_ULOCK, 4) finally: os.close(fd) + +def check_working_xattr(): + fname = str(udir.join('xattr_test0.txt')) + with open(fname, 'wb'): + pass + try: + rposix.setxattr(fname, 'user.foo', '') + except OSError: + return False + else: + return True + + at pytest.mark.skipif(not (hasattr(rposix, 'getxattr') and check_working_xattr()), + reason="Requires working rposix.getxattr()") + at given( + name=st.text( + alphabet=st.characters(min_codepoint=1), min_size=1, max_size=10), + value=st.binary(max_size=10), + follow_symlinks=st.booleans(), use_fd=st.booleans()) +def test_xattr(name, value, follow_symlinks, use_fd): + assume(follow_symlinks or not use_fd) + name = 'user.' + name.encode('utf-8') + fname = str(udir.join('xattr_test.txt')) + try: + os.unlink(fname) + except OSError: + pass + with open(fname, 'wb'): + pass + if use_fd: + file_id = os.open(fname, os.O_CREAT, 0777) + read, write, delete = rposix.fgetxattr, rposix.fsetxattr, rposix.fremovexattr + all_names = rposix.flistxattr + else: + file_id = fname + if follow_symlinks: + read, write, delete = rposix.getxattr, rposix.setxattr, rposix.removexattr + all_names = rposix.listxattr + else: + read = lambda *args, **kwargs: rposix.getxattr(*args, follow_symlinks=False, **kwargs) + write = lambda *args, **kwargs: rposix.setxattr(*args, follow_symlinks=False, **kwargs) + delete = lambda *args, **kwargs: rposix.removexattr(*args, follow_symlinks=False, **kwargs) + all_names = lambda *args, **kwargs: rposix.listxattr(*args, follow_symlinks=False, **kwargs) + try: + init_names = all_names(file_id) + with pytest.raises(OSError): + read(file_id, name) + write(file_id, name, value) + assert read(file_id, name) == value + assert set(all_names(file_id)) == set(init_names + [name]) + assert '' not in all_names(file_id) + delete(file_id, name) + with pytest.raises(OSError): + read(file_id, name) + assert set(all_names(file_id)) == set(init_names) + finally: + if use_fd: + os.close(file_id) From pypy.commits at gmail.com Wed Jan 10 16:18:21 2018 From: pypy.commits at gmail.com (arigo) Date: Wed, 10 Jan 2018 13:18:21 -0800 (PST) Subject: [pypy-commit] cffi default: Issue #352 Message-ID: <5a56831d.88c3df0a.72f8f.b978@mx.google.com> Author: Armin Rigo Branch: Changeset: r3060:d16706e3c2da Date: 2018-01-10 22:18 +0100 http://bitbucket.org/cffi/cffi/changeset/d16706e3c2da/ Log: Issue #352 Oops. diff --git a/c/lib_obj.c b/c/lib_obj.c --- a/c/lib_obj.c +++ b/c/lib_obj.c @@ -505,6 +505,7 @@ return x; missing: + /*** ATTRIBUTEERROR IS SET HERE ***/ p = PyText_AsUTF8(name); if (p == NULL) return NULL; @@ -534,6 +535,7 @@ #if PY_MAJOR_VERSION >= 3 if (strcmp(p, "__loader__") == 0 || strcmp(p, "__spec__") == 0) { /* some more module-like behavior hacks */ + PyErr_Clear(); Py_INCREF(Py_None); return Py_None; } diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py --- a/testing/cffi1/test_recompiler.py +++ b/testing/cffi1/test_recompiler.py @@ -2287,3 +2287,13 @@ def test_char16_char32_plain_c(): test_char16_char32_type(no_cpp=True) + +def test_loader_spec(): + ffi = FFI() + lib = verify(ffi, "test_loader_spec", "") + if sys.version_info < (3,): + assert not hasattr(lib, '__loader__') + assert not hasattr(lib, '__spec__') + else: + assert lib.__loader__ is None + assert lib.__spec__ is None From pypy.commits at gmail.com Wed Jan 10 16:43:30 2018 From: pypy.commits at gmail.com (arigo) Date: Wed, 10 Jan 2018 13:43:30 -0800 (PST) Subject: [pypy-commit] pypy default: Update test_recompiler to match cffi d16706e3c2da Message-ID: <5a568902.4192df0a.114a8.530c@mx.google.com> Author: Armin Rigo Branch: Changeset: r93649:71bbe765aa42 Date: 2018-01-10 22:42 +0100 http://bitbucket.org/pypy/pypy/changeset/71bbe765aa42/ Log: Update test_recompiler to match cffi d16706e3c2da diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py --- a/pypy/module/_cffi_backend/test/test_recompiler.py +++ b/pypy/module/_cffi_backend/test/test_recompiler.py @@ -8,7 +8,8 @@ @unwrap_spec(cdef='text', module_name='text', source='text', packed=int) def prepare(space, cdef, module_name, source, w_includes=None, - w_extra_source=None, w_min_version=None, packed=False): + w_extra_source=None, w_min_version=None, packed=False, + w_extra_compile_args=None): try: import cffi from cffi import FFI # <== the system one, which @@ -55,10 +56,14 @@ sources = [] if w_extra_source is not None: sources.append(space.str_w(w_extra_source)) + kwargs = {} + if w_extra_compile_args is not None: + kwargs['extra_compile_args'] = space.unwrap(w_extra_compile_args) ext = ffiplatform.get_extension(c_file, module_name, include_dirs=[str(rdir)], export_symbols=['_cffi_pypyinit_' + base_module_name], - sources=sources) + sources=sources, + **kwargs) ffiplatform.compile(str(rdir), ext) for extension in ['so', 'pyd', 'dylib']: @@ -2054,3 +2059,51 @@ "Such structs are only supported as return value if the function is " "'API mode' and non-variadic (i.e. declared inside ffibuilder.cdef()" "+ffibuilder.set_source() and not taking a final '...' argument)") + + def test_gcc_visibility_hidden(self): + import sys + if sys.platform == 'win32': + skip("test for gcc/clang") + ffi, lib = self.prepare(""" + int f(int); + """, "test_gcc_visibility_hidden", """ + int f(int a) { return a + 40; } + """, extra_compile_args=['-fvisibility=hidden']) + assert lib.f(2) == 42 + + def test_override_default_definition(self): + ffi, lib = self.prepare(""" + typedef long int16_t, char16_t; + """, "test_override_default_definition", """ + """) + assert ffi.typeof("int16_t") is ffi.typeof("char16_t") is ffi.typeof("long") + + def test_char16_char32_plain_c(self): + ffi, lib = self.prepare(""" + char16_t foo_2bytes(char16_t); + char32_t foo_4bytes(char32_t); + """, "test_char16_char32_type_nocpp", """ + #if !defined(__cplusplus) || (!defined(_LIBCPP_VERSION) && __cplusplus < 201103L) + typedef uint_least16_t char16_t; + typedef uint_least32_t char32_t; + #endif + + char16_t foo_2bytes(char16_t a) { return (char16_t)(a + 42); } + char32_t foo_4bytes(char32_t a) { return (char32_t)(a + 42); } + """, min_version=(1, 11, 0)) + assert lib.foo_2bytes(u'\u1234') == u'\u125e' + assert lib.foo_4bytes(u'\u1234') == u'\u125e' + assert lib.foo_4bytes(u'\U00012345') == u'\U0001236f' + raises(TypeError, lib.foo_2bytes, u'\U00012345') + raises(TypeError, lib.foo_2bytes, 1234) + raises(TypeError, lib.foo_4bytes, 1234) + + def test_loader_spec(self): + import sys + ffi, lib = self.prepare("", "test_loader_spec", "") + if sys.version_info < (3,): + assert not hasattr(lib, '__loader__') + assert not hasattr(lib, '__spec__') + else: + assert lib.__loader__ is None + assert lib.__spec__ is None From pypy.commits at gmail.com Wed Jan 10 17:21:19 2018 From: pypy.commits at gmail.com (arigo) Date: Wed, 10 Jan 2018 14:21:19 -0800 (PST) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <5a5691df.4dafdf0a.af0f0.9b7d@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r93650:52e62b9d7e92 Date: 2018-01-10 22:44 +0100 http://bitbucket.org/pypy/pypy/changeset/52e62b9d7e92/ Log: hg merge default diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py --- a/pypy/module/_cffi_backend/test/test_recompiler.py +++ b/pypy/module/_cffi_backend/test/test_recompiler.py @@ -8,7 +8,8 @@ @unwrap_spec(cdef='text', module_name='text', source='text', packed=int) def prepare(space, cdef, module_name, source, w_includes=None, - w_extra_source=None, w_min_version=None, packed=False): + w_extra_source=None, w_min_version=None, packed=False, + w_extra_compile_args=None): try: import cffi from cffi import FFI # <== the system one, which @@ -55,10 +56,14 @@ sources = [] if w_extra_source is not None: sources.append(space.str_w(w_extra_source)) + kwargs = {} + if w_extra_compile_args is not None: + kwargs['extra_compile_args'] = space.unwrap(w_extra_compile_args) ext = ffiplatform.get_extension(c_file, module_name, include_dirs=[str(rdir)], export_symbols=['_cffi_pypyinit_' + base_module_name], - sources=sources) + sources=sources, + **kwargs) ffiplatform.compile(str(rdir), ext) for extension in ['so', 'pyd', 'dylib']: @@ -2055,3 +2060,51 @@ "Such structs are only supported as return value if the function is " "'API mode' and non-variadic (i.e. declared inside ffibuilder.cdef()" "+ffibuilder.set_source() and not taking a final '...' argument)") + + def test_gcc_visibility_hidden(self): + import sys + if sys.platform == 'win32': + skip("test for gcc/clang") + ffi, lib = self.prepare(""" + int f(int); + """, "test_gcc_visibility_hidden", """ + int f(int a) { return a + 40; } + """, extra_compile_args=['-fvisibility=hidden']) + assert lib.f(2) == 42 + + def test_override_default_definition(self): + ffi, lib = self.prepare(""" + typedef long int16_t, char16_t; + """, "test_override_default_definition", """ + """) + assert ffi.typeof("int16_t") is ffi.typeof("char16_t") is ffi.typeof("long") + + def test_char16_char32_plain_c(self): + ffi, lib = self.prepare(""" + char16_t foo_2bytes(char16_t); + char32_t foo_4bytes(char32_t); + """, "test_char16_char32_type_nocpp", """ + #if !defined(__cplusplus) || (!defined(_LIBCPP_VERSION) && __cplusplus < 201103L) + typedef uint_least16_t char16_t; + typedef uint_least32_t char32_t; + #endif + + char16_t foo_2bytes(char16_t a) { return (char16_t)(a + 42); } + char32_t foo_4bytes(char32_t a) { return (char32_t)(a + 42); } + """, min_version=(1, 11, 0)) + assert lib.foo_2bytes(u'\u1234') == u'\u125e' + assert lib.foo_4bytes(u'\u1234') == u'\u125e' + assert lib.foo_4bytes(u'\U00012345') == u'\U0001236f' + raises(TypeError, lib.foo_2bytes, u'\U00012345') + raises(TypeError, lib.foo_2bytes, 1234) + raises(TypeError, lib.foo_4bytes, 1234) + + def test_loader_spec(self): + import sys + ffi, lib = self.prepare("", "test_loader_spec", "") + if sys.version_info < (3,): + assert not hasattr(lib, '__loader__') + assert not hasattr(lib, '__spec__') + else: + assert lib.__loader__ is None + assert lib.__spec__ is None From pypy.commits at gmail.com Wed Jan 10 17:24:10 2018 From: pypy.commits at gmail.com (arigo) Date: Wed, 10 Jan 2018 14:24:10 -0800 (PST) Subject: [pypy-commit] pypy default: update to cffi/d16706e3c2da Message-ID: <5a56928a.0ba3df0a.4fe9e.4b30@mx.google.com> Author: Armin Rigo Branch: Changeset: r93651:9651b48098f7 Date: 2018-01-10 23:23 +0100 http://bitbucket.org/pypy/pypy/changeset/9651b48098f7/ Log: update to cffi/d16706e3c2da diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h --- a/lib_pypy/cffi/_cffi_include.h +++ b/lib_pypy/cffi/_cffi_include.h @@ -7,11 +7,38 @@ we can learn about Py_DEBUG from pyconfig.h, but it is unclear if the same works for the other two macros. Py_DEBUG implies them, but not the other way around. + + Issue #350: more mess: on Windows, with _MSC_VER, we have to define + Py_LIMITED_API even before including pyconfig.h. In that case, we + guess what pyconfig.h will do to the macros above, and check our + guess after the #include. */ #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) -# include -# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) -# define Py_LIMITED_API +# ifdef _MSC_VER +# if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API +# endif +# include + /* sanity-check: Py_LIMITED_API will cause crashes if any of these + are also defined. Normally, the Python file PC/pyconfig.h does not + cause any of these to be defined, with the exception that _DEBUG + causes Py_DEBUG. Double-check that. */ +# ifdef Py_LIMITED_API +# if defined(Py_DEBUG) +# error "pyconfig.h unexpectedly defines Py_DEBUG but _DEBUG is not set" +# endif +# if defined(Py_TRACE_REFS) +# error "pyconfig.h unexpectedly defines Py_TRACE_REFS" +# endif +# if defined(Py_REF_DEBUG) +# error "pyconfig.h unexpectedly defines Py_REF_DEBUG" +# endif +# endif +# else +# include +# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API +# endif # endif #endif diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -295,8 +295,9 @@ base_module_name = self.module_name.split('.')[-1] if self.ffi._embedding is not None: prnt('#define _CFFI_MODULE_NAME "%s"' % (self.module_name,)) - prnt('#define _CFFI_PYTHON_STARTUP_CODE %s' % - (self._string_literal(self.ffi._embedding),)) + prnt('static const char _CFFI_PYTHON_STARTUP_CODE[] = {') + self._print_string_literal_in_array(self.ffi._embedding) + prnt('0 };') prnt('#ifdef PYPY_VERSION') prnt('# define _CFFI_PYTHON_STARTUP_FUNC _cffi_pypyinit_%s' % ( base_module_name,)) @@ -1271,17 +1272,18 @@ _generate_cpy_extern_python_plus_c_ctx = \ _generate_cpy_extern_python_ctx - def _string_literal(self, s): - def _char_repr(c): - # escape with a '\' the characters '\', '"' or (for trigraphs) '?' - if c in '\\"?': return '\\' + c - if ' ' <= c < '\x7F': return c - if c == '\n': return '\\n' - return '\\%03o' % ord(c) - lines = [] - for line in s.splitlines(True) or ['']: - lines.append('"%s"' % ''.join([_char_repr(c) for c in line])) - return ' \\\n'.join(lines) + def _print_string_literal_in_array(self, s): + prnt = self._prnt + prnt('// # NB. this is not a string because of a size limit in MSVC') + for line in s.splitlines(True): + prnt(('// ' + line).rstrip()) + printed_line = '' + for c in line: + if len(printed_line) >= 76: + prnt(printed_line) + printed_line = '' + printed_line += '%d,' % (ord(c),) + prnt(printed_line) # ---------- # emitting the opcodes for individual types diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py --- a/lib_pypy/cffi/verifier.py +++ b/lib_pypy/cffi/verifier.py @@ -301,7 +301,6 @@ return suffixes def _ensure_dir(filename): - try: - os.makedirs(os.path.dirname(filename)) - except OSError: - pass + dirname = os.path.dirname(filename) + if dirname and not os.path.isdir(dirname): + os.makedirs(dirname) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py @@ -2288,3 +2288,13 @@ def test_char16_char32_plain_c(): test_char16_char32_type(no_cpp=True) + +def test_loader_spec(): + ffi = FFI() + lib = verify(ffi, "test_loader_spec", "") + if sys.version_info < (3,): + assert not hasattr(lib, '__loader__') + assert not hasattr(lib, '__spec__') + else: + assert lib.__loader__ is None + assert lib.__spec__ is None From pypy.commits at gmail.com Wed Jan 10 17:32:48 2018 From: pypy.commits at gmail.com (arigo) Date: Wed, 10 Jan 2018 14:32:48 -0800 (PST) Subject: [pypy-commit] cffi default: What's new in 1.11.3 Message-ID: <5a569490.068a1c0a.1bc36.dfbc@mx.google.com> Author: Armin Rigo Branch: Changeset: r3062:0f751c6b375a Date: 2018-01-10 23:32 +0100 http://bitbucket.org/cffi/cffi/changeset/0f751c6b375a/ Log: What's new in 1.11.3 diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst --- a/doc/source/whatsnew.rst +++ b/doc/source/whatsnew.rst @@ -3,6 +3,16 @@ ====================== +v1.11.3 +======= + +* More Windows fixes: MSVC does not support large literal strings in C + code (from ``ffi.embedding_init_code(large_string)``); and a MSVC-only + issue with ``Py_LIMITED_API`` on CPython 3.x. + +* Other misc. issues and documentation improvements + + v1.11.2 ======= From pypy.commits at gmail.com Wed Jan 10 17:32:46 2018 From: pypy.commits at gmail.com (arigo) Date: Wed, 10 Jan 2018 14:32:46 -0800 (PST) Subject: [pypy-commit] cffi default: Bump version to 1.11.3 Message-ID: <5a56948e.f285df0a.d1012.6b9d@mx.google.com> Author: Armin Rigo Branch: Changeset: r3061:f4f452929690 Date: 2018-01-10 23:26 +0100 http://bitbucket.org/cffi/cffi/changeset/f4f452929690/ Log: Bump version to 1.11.3 diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -2,7 +2,7 @@ #include #include "structmember.h" -#define CFFI_VERSION "1.11.2" +#define CFFI_VERSION "1.11.3" #ifdef MS_WIN32 #include diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -12,7 +12,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.11.2", ("This test_c.py file is for testing a version" +assert __version__ == "1.11.3", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/cffi/__init__.py b/cffi/__init__.py --- a/cffi/__init__.py +++ b/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.11.2" -__version_info__ = (1, 11, 2) +__version__ = "1.11.3" +__version_info__ = (1, 11, 3) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/cffi/_embedding.h b/cffi/_embedding.h --- a/cffi/_embedding.h +++ b/cffi/_embedding.h @@ -247,7 +247,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.11.2" + "\ncompiled with cffi version: 1.11.3" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/doc/source/conf.py b/doc/source/conf.py --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -47,7 +47,7 @@ # The short X.Y version. version = '1.11' # The full version, including alpha/beta/rc tags. -release = '1.11.2' +release = '1.11.3' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/installation.rst b/doc/source/installation.rst --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -53,13 +53,13 @@ * https://pypi.python.org/pypi/cffi -* Checksums of the "source" package version 1.11.2: +* Checksums of the "source" package version 1.11.3: - - MD5: a731487324b501c8295221b629d3f5f3 + - MD5: ... - - SHA: 04d2df85eb1921630b4f9206886737eb37200c19 + - SHA: ... - - SHA256: ab87dd91c0c4073758d07334c1e5f712ce8fe48f007b86f8238773963ee700a6 + - SHA256: ... * Or grab the most current version from the `Bitbucket page`_: ``hg clone https://bitbucket.org/cffi/cffi`` diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -186,7 +186,7 @@ `Mailing list `_ """, - version='1.11.2', + version='1.11.3', packages=['cffi'] if cpython else [], package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', '_embedding.h', '_cffi_errors.h']} From pypy.commits at gmail.com Wed Jan 10 23:43:43 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 10 Jan 2018 20:43:43 -0800 (PST) Subject: [pypy-commit] pypy default: fix merge from f560bd567d72 Message-ID: <5a56eb7f.0490df0a.adce2.8f46@mx.google.com> Author: Matti Picus Branch: Changeset: r93652:d1ea0800978f Date: 2018-01-11 06:42 +0200 http://bitbucket.org/pypy/pypy/changeset/d1ea0800978f/ Log: fix merge from f560bd567d72 diff --git a/rpython/rlib/rwinreg.py b/rpython/rlib/rwinreg.py --- a/rpython/rlib/rwinreg.py +++ b/rpython/rlib/rwinreg.py @@ -47,7 +47,7 @@ HKEY = rwin32.HANDLE PHKEY = rffi.CArrayPtr(HKEY) REGSAM = rwin32.DWORD -suffix = 'W' +suffix = 'A' RegSetValue = external( 'RegSetValue' + suffix, From pypy.commits at gmail.com Thu Jan 11 03:14:12 2018 From: pypy.commits at gmail.com (Hiroshi Hatake) Date: Thu, 11 Jan 2018 00:14:12 -0800 (PST) Subject: [pypy-commit] cffi fix-typo: Fix a typo in cdef.rst Message-ID: <5a571cd4.f4acdf0a.7e296.b9d8@mx.google.com> Author: Hiroshi Hatake Branch: fix-typo Changeset: r3063:fd9081cfd060 Date: 2018-01-10 16:01 +0900 http://bitbucket.org/cffi/cffi/changeset/fd9081cfd060/ Log: Fix a typo in cdef.rst intented → intended diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst --- a/doc/source/cdef.rst +++ b/doc/source/cdef.rst @@ -701,7 +701,7 @@ ------------------------------- A few C libraries are actually hard to use correctly in a ``dlopen()`` -setting. This is because most C libraries are intented for, and tested +setting. This is because most C libraries are intended for, and tested with, a situation where they are *linked* with another program, using either static linking or dynamic linking --- but from a program written in C, at start-up, using the linker's capabilities instead of From pypy.commits at gmail.com Thu Jan 11 03:14:14 2018 From: pypy.commits at gmail.com (Hiroshi Hatake) Date: Thu, 11 Jan 2018 00:14:14 -0800 (PST) Subject: [pypy-commit] cffi fix-typo: Fix a typo in installation Message-ID: <5a571cd6.c6a0df0a.da083.467c@mx.google.com> Author: Hiroshi Hatake Branch: fix-typo Changeset: r3064:c597a8bf02bf Date: 2018-01-10 16:01 +0900 http://bitbucket.org/cffi/cffi/changeset/c597a8bf02bf/ Log: Fix a typo in installation Aternatively → Alternatively diff --git a/doc/source/installation.rst b/doc/source/installation.rst --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -114,7 +114,7 @@ PKG_CONFIG_PATH=/usr/local/opt/libffi/lib/pkgconfig pip install cffi -Aternatively, **on OS/X 10.6** (Thanks Juraj Sukop for this) +Alternatively, **on OS/X 10.6** (Thanks Juraj Sukop for this) For building libffi you can use the default install path, but then, in ``setup.py`` you need to change:: From pypy.commits at gmail.com Thu Jan 11 03:14:08 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 11 Jan 2018 00:14:08 -0800 (PST) Subject: [pypy-commit] cffi default: Merged in cosmo0920/cffi/fix-typo (pull request #84) Message-ID: <5a571cd0.eb8ddf0a.f8cbc.8532@mx.google.com> Author: Armin Rigo Branch: Changeset: r3066:3dcec7271e97 Date: 2018-01-11 08:14 +0000 http://bitbucket.org/cffi/cffi/changeset/3dcec7271e97/ Log: Merged in cosmo0920/cffi/fix-typo (pull request #84) Fix typo diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst --- a/doc/source/cdef.rst +++ b/doc/source/cdef.rst @@ -701,7 +701,7 @@ ------------------------------- A few C libraries are actually hard to use correctly in a ``dlopen()`` -setting. This is because most C libraries are intented for, and tested +setting. This is because most C libraries are intended for, and tested with, a situation where they are *linked* with another program, using either static linking or dynamic linking --- but from a program written in C, at start-up, using the linker's capabilities instead of diff --git a/doc/source/installation.rst b/doc/source/installation.rst --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -114,7 +114,7 @@ PKG_CONFIG_PATH=/usr/local/opt/libffi/lib/pkgconfig pip install cffi -Aternatively, **on OS/X 10.6** (Thanks Juraj Sukop for this) +Alternatively, **on OS/X 10.6** (Thanks Juraj Sukop for this) For building libffi you can use the default install path, but then, in ``setup.py`` you need to change:: diff --git a/doc/source/using.rst b/doc/source/using.rst --- a/doc/source/using.rst +++ b/doc/source/using.rst @@ -28,7 +28,7 @@ *New in version 1.11:* in addition to ``wchar_t``, the C types ``char16_t`` and ``char32_t`` work the same but with a known fixed size. In previous versions, this could be achieved using ``uint16_t`` and -``int32_t`` but without automatic convertion to Python unicodes. +``int32_t`` but without automatic conversion to Python unicodes. Pointers, structures and arrays are more complex: they don't have an obvious Python equivalent. Thus, they correspond to objects of type From pypy.commits at gmail.com Thu Jan 11 03:14:16 2018 From: pypy.commits at gmail.com (Hiroshi Hatake) Date: Thu, 11 Jan 2018 00:14:16 -0800 (PST) Subject: [pypy-commit] cffi fix-typo: Fix a typo in using.rst Message-ID: <5a571cd8.4fc11c0a.5cc18.07fe@mx.google.com> Author: Hiroshi Hatake Branch: fix-typo Changeset: r3065:74d85eb3df35 Date: 2018-01-10 16:02 +0900 http://bitbucket.org/cffi/cffi/changeset/74d85eb3df35/ Log: Fix a typo in using.rst convertion → conversion diff --git a/doc/source/using.rst b/doc/source/using.rst --- a/doc/source/using.rst +++ b/doc/source/using.rst @@ -28,7 +28,7 @@ *New in version 1.11:* in addition to ``wchar_t``, the C types ``char16_t`` and ``char32_t`` work the same but with a known fixed size. In previous versions, this could be achieved using ``uint16_t`` and -``int32_t`` but without automatic convertion to Python unicodes. +``int32_t`` but without automatic conversion to Python unicodes. Pointers, structures and arrays are more complex: they don't have an obvious Python equivalent. Thus, they correspond to objects of type From pypy.commits at gmail.com Thu Jan 11 14:58:53 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 11 Jan 2018 11:58:53 -0800 (PST) Subject: [pypy-commit] pypy default: Added tag release-pypy3.5-v5.10.1 for changeset 3f6eaa010fce Message-ID: <5a57c1fd.3886df0a.8c2ca.20c5@mx.google.com> Author: Matti Picus Branch: Changeset: r93653:db878ccf6890 Date: 2018-01-11 21:57 +0200 http://bitbucket.org/pypy/pypy/changeset/db878ccf6890/ Log: Added tag release-pypy3.5-v5.10.1 for changeset 3f6eaa010fce diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -50,3 +50,4 @@ 0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0 0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0 09f9160b643e3f02ccb8c843b2fbb4e5cbf54082 release-pypy3.5-v5.10.0 +3f6eaa010fce78cc7973bdc1dfdb95970f08fed2 release-pypy3.5-v5.10.1 From pypy.commits at gmail.com Thu Jan 11 15:03:00 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 11 Jan 2018 12:03:00 -0800 (PST) Subject: [pypy-commit] cffi default: Expand Message-ID: <5a57c2f4.12711c0a.8a962.7683@mx.google.com> Author: Armin Rigo Branch: Changeset: r3067:ab8662981c0d Date: 2018-01-11 21:02 +0100 http://bitbucket.org/cffi/cffi/changeset/ab8662981c0d/ Log: Expand diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst --- a/doc/source/whatsnew.rst +++ b/doc/source/whatsnew.rst @@ -6,11 +6,18 @@ v1.11.3 ======= -* More Windows fixes: MSVC does not support large literal strings in C - code (from ``ffi.embedding_init_code(large_string)``); and a MSVC-only - issue with ``Py_LIMITED_API`` on CPython 3.x. +* Fix on CPython 3.x: reading the attributes ``__loader__`` or + ``__spec__`` from the cffi-generated lib modules gave a buggy + SystemError. (These attributes are always None, and provided only to + help compatibility with tools that expect them in all modules.) -* Other misc. issues and documentation improvements +* More Windows fixes: workaround for MSVC not supporting large + literal strings in C code (from + ``ffi.embedding_init_code(large_string)``); and an issue with + ``Py_LIMITED_API`` linking with ``python35.dll/python36.dll`` instead + of ``python3.dll``. + +* Small documentation improvements. v1.11.2 From pypy.commits at gmail.com Thu Jan 11 15:28:49 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 11 Jan 2018 12:28:49 -0800 (PST) Subject: [pypy-commit] pypy.org extradoc: add sha256 hashes for pypy3.5-v5.10.1, note s390x, maxos64 sierra are missing Message-ID: <5a57c901.cd5c1c0a.8628a.9504@mx.google.com> Author: Matti Picus Branch: extradoc Changeset: r912:39d20dbad619 Date: 2018-01-11 22:28 +0200 http://bitbucket.org/pypy/pypy.org/changeset/39d20dbad619/ Log: add sha256 hashes for pypy3.5-v5.10.1, note s390x, maxos64 sierra are missing diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -17,12 +17,12 @@ * the Python2.7 compatible release — **PyPy2.7 v5.10.0** — (`what's new in PyPy2.7?`_) -* the Python3.5 compatible beta quality release — **PyPy3.5 v5.10.0** — (`what's new in PyPy3.5?`_). +* the Python3.5 compatible release — **PyPy3.5 v5.10.1** — (`what's new in PyPy3.5?`_). * the Python2.7 Software Transactional Memory special release — **PyPy-STM 2.5.1** (Linux x86-64 only) .. _what's new in PyPy2.7?: http://doc.pypy.org/en/latest/release-v5.10.0.html -.. _what's new in PyPy3.5?: http://doc.pypy.org/en/latest/release-v5.10.0.html +.. _what's new in PyPy3.5?: http://doc.pypy.org/en/latest/release-v5.10.1.html .. class:: download_menu @@ -115,8 +115,8 @@ .. _mirror: http://buildbot.pypy.org/mirror/ .. _FreshPorts: http://www.freshports.org/lang/pypy -Python 3.5.3 compatible PyPy3.5 v5.10 -------------------------------------- +Python 3.5.3 compatible PyPy3.5 v5.10.1 +--------------------------------------- .. class:: download_menu @@ -128,24 +128,24 @@ * `ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Raspbian)`__ (see ``[1]`` below) * `ARM Softfloat Linux binary (ARMEL/gnueabi, tar.bz2, Ubuntu Precise)`__ (see ``[1]`` below) * `Mac OS X binary (64bit)`__ (High Sierra) -* `Mac OS X binary (64bit) (2)`__ (Sierra and below) +* `Mac OS X binary (64bit) for 5.10.0 (2)`__ (Sierra and below) * `Windows binary (32bit)`__ (you might need the VS 2008 runtime library installer `vcredist_x86.exe`_.) -* `s390x Linux binary (tar.bz2 built on Redhat Linux 7.2)`__ (see ``[1]`` below) +* `s390x Linux binary for 5.10.0 (tar.bz2 built on Redhat Linux 7.2)`__ (see ``[1]`` below) * `Source (tar.bz2)`__; `Source (zip)`__. See below for more about the sources. * `All our downloads,`__ including previous versions. We also have a mirror_, but please use only if you have troubles accessing the links above -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.0-linux32.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.0-linux64.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.0-linux-armhf-raspbian.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.0-linux-armel.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.0-osx64.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.1-linux32.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.1-linux64.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.1-linux-armhf-raspbian.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.1-linux-armel.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.1-osx64.tar.bz2 .. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.0-osx64-2.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.0-win32.zip +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.1-win32.zip .. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.0-s390x.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.0-src.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.0-src.zip +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.1-src.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.1-src.zip .. __: https://bitbucket.org/pypy/pypy/downloads If your CPU is really, really old, it may be a x86-32 without SSE2. @@ -307,9 +307,9 @@ Alternatively, the following smaller package contains the source at the same revision as the above binaries: - * `pypy2-v5.10.0-src.tar.bz2`__ (sources) + * `pypy2-v5.10.1-src.tar.bz2`__ (sources) - .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.10.0-src.tar.bz2 + .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.10.1-src.tar.bz2 2. Make sure you **installed the dependencies.** See the list here__. @@ -466,12 +466,6 @@ 9afa1a36a5fc55ebc3e80576f05f44294f2b0de279862286fe00f5ee139965b1 pypy2-v5.10.0-ppc64.tar.bz2 2c32ccfa80e3e2ec56b4cc848526046d7b0de1f2f1a92b0cedeb414ec76745ab pypy2-v5.10.0-ppc64le.tar.bz2 -pypy 3.5-v5.9.0 sha256:: - - d8c41ede3758127718944cc2fd6bf78ed4303d946f85596cac91281ccce36165 pypy3-v5.9.0-linux64.tar.bz2 - a014f47f50a1480f871a0b82705f904b38c93c4ca069850eb37653fedafb1b97 pypy3-v5.9.0-src.tar.bz2 - c5d7fa206cdf425de3950ef8ff578deb0745a723b69b64bb121210a5b8df8c65 pypy3-v5.9.0-src.zip - pypy 3.5-v5.10.0 sha256:: 529bc3b11edbdcdd676d90c805b8f607f6eedd5f0ec457a31bbe09c03f5bebfe pypy3-v5.10.0-linux32.tar.bz2 @@ -486,3 +480,14 @@ 2d93bf2bd7b1d031b96331d3fde6cacdda95673ce6875d6d1669c4c0ea2a52bc pypy3-v5.10.0-win32.zip +pypy 3.5-v5.10.0 sha256:: + + a6ceca9ee5dc511de7902164464b88311fec9366c5673d0c00528eda862bbe54 pypy3-v5.10.1-linux32.tar.bz2 + 75a276e1ee1863967bbacb70c5bff636de200768c0ec90e72f7ec17aace0aefe pypy3-v5.10.1-linux64.tar.bz2 + 5065e9ad958d06b9612ba974f43997d20168d4245c054dd43270e4b458782282 pypy3-v5.10.1-linux-armel.tar.bz2 + 203dd595fbad7055340b23326f20c85b0d6c11c4877e3559a437611fc2ac40c2 pypy3-v5.10.1-linux-armhf-raspbian.tar.bz2 + 52f006611513c995fdebba6e72d394186d4085460408cbbe086e5467bf3fb9b6 pypy3-v5.10.1-osx64.tar.bz2 + f5548e06e2fc0c24ec8b6e3c5b09f90081818f7caa3e436dc312592611724713 pypy3-v5.10.1-src.tar.bz2 + 182378d7aab395ee6cf539fb011ec0e384624282834aaaed4a663972a5aa8797 pypy3-v5.10.1-src.zip + 4edf4f021689a529e5a631c5cca72a1a9dc19a6ea2091e64289cdd5b60eaf929 pypy3-v5.10.1-win32.zip + From pypy.commits at gmail.com Thu Jan 11 15:48:57 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 11 Jan 2018 12:48:57 -0800 (PST) Subject: [pypy-commit] cffi default: hg merge release-1.11 Message-ID: <5a57cdb9.410e1c0a.15471.903f@mx.google.com> Author: Armin Rigo Branch: Changeset: r3070:1cdf364307f9 Date: 2018-01-11 21:48 +0100 http://bitbucket.org/cffi/cffi/changeset/1cdf364307f9/ Log: hg merge release-1.11 diff --git a/doc/source/installation.rst b/doc/source/installation.rst --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -55,11 +55,11 @@ * Checksums of the "source" package version 1.11.3: - - MD5: ... + - MD5: 3d263ec84bb9ad08e0edf2d21312fa1e - - SHA: ... + - SHA: 26a73cc075064cc7cd59ec6c58ca0684d16c4ece - - SHA256: ... + - SHA256: 150708ce9e417858f9a4056b5f364d8c7077fd979b9e35307c2d8a4a8e991fd2 * Or grab the most current version from the `Bitbucket page`_: ``hg clone https://bitbucket.org/cffi/cffi`` From pypy.commits at gmail.com Thu Jan 11 15:48:52 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 11 Jan 2018 12:48:52 -0800 (PST) Subject: [pypy-commit] cffi release-1.11: hg merge default Message-ID: <5a57cdb4.4bb7df0a.53291.1fdc@mx.google.com> Author: Armin Rigo Branch: release-1.11 Changeset: r3068:b6c9e34d098f Date: 2018-01-11 21:43 +0100 http://bitbucket.org/cffi/cffi/changeset/b6c9e34d098f/ Log: hg merge default diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -2,7 +2,7 @@ #include #include "structmember.h" -#define CFFI_VERSION "1.11.2" +#define CFFI_VERSION "1.11.3" #ifdef MS_WIN32 #include diff --git a/c/lib_obj.c b/c/lib_obj.c --- a/c/lib_obj.c +++ b/c/lib_obj.c @@ -505,6 +505,7 @@ return x; missing: + /*** ATTRIBUTEERROR IS SET HERE ***/ p = PyText_AsUTF8(name); if (p == NULL) return NULL; @@ -534,6 +535,7 @@ #if PY_MAJOR_VERSION >= 3 if (strcmp(p, "__loader__") == 0 || strcmp(p, "__spec__") == 0) { /* some more module-like behavior hacks */ + PyErr_Clear(); Py_INCREF(Py_None); return Py_None; } diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -12,7 +12,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.11.2", ("This test_c.py file is for testing a version" +assert __version__ == "1.11.3", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/cffi/__init__.py b/cffi/__init__.py --- a/cffi/__init__.py +++ b/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.11.2" -__version_info__ = (1, 11, 2) +__version__ = "1.11.3" +__version_info__ = (1, 11, 3) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h --- a/cffi/_cffi_include.h +++ b/cffi/_cffi_include.h @@ -7,11 +7,38 @@ we can learn about Py_DEBUG from pyconfig.h, but it is unclear if the same works for the other two macros. Py_DEBUG implies them, but not the other way around. + + Issue #350: more mess: on Windows, with _MSC_VER, we have to define + Py_LIMITED_API even before including pyconfig.h. In that case, we + guess what pyconfig.h will do to the macros above, and check our + guess after the #include. */ #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) -# include -# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) -# define Py_LIMITED_API +# ifdef _MSC_VER +# if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API +# endif +# include + /* sanity-check: Py_LIMITED_API will cause crashes if any of these + are also defined. Normally, the Python file PC/pyconfig.h does not + cause any of these to be defined, with the exception that _DEBUG + causes Py_DEBUG. Double-check that. */ +# ifdef Py_LIMITED_API +# if defined(Py_DEBUG) +# error "pyconfig.h unexpectedly defines Py_DEBUG but _DEBUG is not set" +# endif +# if defined(Py_TRACE_REFS) +# error "pyconfig.h unexpectedly defines Py_TRACE_REFS" +# endif +# if defined(Py_REF_DEBUG) +# error "pyconfig.h unexpectedly defines Py_REF_DEBUG" +# endif +# endif +# else +# include +# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API +# endif # endif #endif diff --git a/cffi/_embedding.h b/cffi/_embedding.h --- a/cffi/_embedding.h +++ b/cffi/_embedding.h @@ -247,7 +247,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.11.2" + "\ncompiled with cffi version: 1.11.3" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/cffi/recompiler.py b/cffi/recompiler.py --- a/cffi/recompiler.py +++ b/cffi/recompiler.py @@ -295,8 +295,9 @@ base_module_name = self.module_name.split('.')[-1] if self.ffi._embedding is not None: prnt('#define _CFFI_MODULE_NAME "%s"' % (self.module_name,)) - prnt('#define _CFFI_PYTHON_STARTUP_CODE %s' % - (self._string_literal(self.ffi._embedding),)) + prnt('static const char _CFFI_PYTHON_STARTUP_CODE[] = {') + self._print_string_literal_in_array(self.ffi._embedding) + prnt('0 };') prnt('#ifdef PYPY_VERSION') prnt('# define _CFFI_PYTHON_STARTUP_FUNC _cffi_pypyinit_%s' % ( base_module_name,)) @@ -1271,17 +1272,18 @@ _generate_cpy_extern_python_plus_c_ctx = \ _generate_cpy_extern_python_ctx - def _string_literal(self, s): - def _char_repr(c): - # escape with a '\' the characters '\', '"' or (for trigraphs) '?' - if c in '\\"?': return '\\' + c - if ' ' <= c < '\x7F': return c - if c == '\n': return '\\n' - return '\\%03o' % ord(c) - lines = [] - for line in s.splitlines(True) or ['']: - lines.append('"%s"' % ''.join([_char_repr(c) for c in line])) - return ' \\\n'.join(lines) + def _print_string_literal_in_array(self, s): + prnt = self._prnt + prnt('// # NB. this is not a string because of a size limit in MSVC') + for line in s.splitlines(True): + prnt(('// ' + line).rstrip()) + printed_line = '' + for c in line: + if len(printed_line) >= 76: + prnt(printed_line) + printed_line = '' + printed_line += '%d,' % (ord(c),) + prnt(printed_line) # ---------- # emitting the opcodes for individual types diff --git a/cffi/verifier.py b/cffi/verifier.py --- a/cffi/verifier.py +++ b/cffi/verifier.py @@ -301,7 +301,6 @@ return suffixes def _ensure_dir(filename): - try: - os.makedirs(os.path.dirname(filename)) - except OSError: - pass + dirname = os.path.dirname(filename) + if dirname and not os.path.isdir(dirname): + os.makedirs(dirname) diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst --- a/doc/source/cdef.rst +++ b/doc/source/cdef.rst @@ -701,7 +701,7 @@ ------------------------------- A few C libraries are actually hard to use correctly in a ``dlopen()`` -setting. This is because most C libraries are intented for, and tested +setting. This is because most C libraries are intended for, and tested with, a situation where they are *linked* with another program, using either static linking or dynamic linking --- but from a program written in C, at start-up, using the linker's capabilities instead of diff --git a/doc/source/conf.py b/doc/source/conf.py --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -47,7 +47,7 @@ # The short X.Y version. version = '1.11' # The full version, including alpha/beta/rc tags. -release = '1.11.2' +release = '1.11.3' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/installation.rst b/doc/source/installation.rst --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -53,13 +53,13 @@ * https://pypi.python.org/pypi/cffi -* Checksums of the "source" package version 1.11.2: +* Checksums of the "source" package version 1.11.3: - - MD5: a731487324b501c8295221b629d3f5f3 + - MD5: ... - - SHA: 04d2df85eb1921630b4f9206886737eb37200c19 + - SHA: ... - - SHA256: ab87dd91c0c4073758d07334c1e5f712ce8fe48f007b86f8238773963ee700a6 + - SHA256: ... * Or grab the most current version from the `Bitbucket page`_: ``hg clone https://bitbucket.org/cffi/cffi`` @@ -114,7 +114,7 @@ PKG_CONFIG_PATH=/usr/local/opt/libffi/lib/pkgconfig pip install cffi -Aternatively, **on OS/X 10.6** (Thanks Juraj Sukop for this) +Alternatively, **on OS/X 10.6** (Thanks Juraj Sukop for this) For building libffi you can use the default install path, but then, in ``setup.py`` you need to change:: diff --git a/doc/source/overview.rst b/doc/source/overview.rst --- a/doc/source/overview.rst +++ b/doc/source/overview.rst @@ -45,8 +45,9 @@ there, %s!\n"``. In general it is ``somestring.encode(myencoding)``. *Python 3 on Windows:* ``ffi.dlopen(None)`` does not work. This problem -is messy and not really fixable. The example above could be fixed by -calling another function from a specific DLL that exists on your system. +is messy and not really fixable. The problem does not occur if you try +to call a fucntion from a specific DLL that exists on your system: then +you use ``ffi.dlopen("path.dll")``. *This example does not call any C compiler. It works in the so-called ABI mode, which means that it will crash if you call some function or @@ -76,20 +77,27 @@ ffibuilder = FFI() ffibuilder.set_source("_example", - r""" // passed to the real C compiler + r""" // passed to the real C compiler, + // contains implementation of things declared in cdef() #include #include + + struct passwd *get_pw_for_root(void) { + return getpwuid(0); + } """, libraries=[]) # or a list of libraries to link with # (more arguments like setup.py's Extension class: # include_dirs=[..], extra_objects=[..], and so on) - ffibuilder.cdef(""" // some declarations from the man page + ffibuilder.cdef(""" + // declarations that are shared between Python and C struct passwd { char *pw_name; ...; // literally dot-dot-dot }; - struct passwd *getpwuid(int uid); + struct passwd *getpwuid(int uid); // defined in + struct passwd *get_pw_for_root(void); // defined in set_source() """) if __name__ == "__main__": @@ -113,15 +121,18 @@ p = lib.getpwuid(0) assert ffi.string(p.pw_name) == b'root' + p = lib.get_pw_for_root() + assert ffi.string(p.pw_name) == b'root' Note that this works independently of the exact C layout of ``struct passwd`` (it is "API level", as opposed to "ABI level"). It requires a C compiler in order to run ``example_build.py``, but it is much more portable than trying to get the details of the fields of ``struct -passwd`` exactly right. Similarly, we declared ``getpwuid()`` as -taking an ``int`` argument. On some platforms this might be slightly -incorrect---but it does not matter. It is also faster than the ABI -mode. +passwd`` exactly right. Similarly, in the ``cdef()`` we declared +``getpwuid()`` as taking an ``int`` argument; on some platforms this +might be slightly incorrect---but it does not matter. + +Note also that at runtime, the API mode is faster than the ABI mode. To integrate it inside a ``setup.py`` distribution with Setuptools: diff --git a/doc/source/ref.rst b/doc/source/ref.rst --- a/doc/source/ref.rst +++ b/doc/source/ref.rst @@ -401,7 +401,12 @@ nothing gets out of sync if the size estimates do not match. It only makes the next GC start more or less early. +Note that if you have several ``ffi.gc()`` objects, the corresponding +destructors will be called in a random order. If you need a particular +order, see the discussion in `issue 340`__. + .. __: http://bugs.python.org/issue31105 +.. __: https://bitbucket.org/cffi/cffi/issues/340/resources-release-issues .. _ffi-new-handle: diff --git a/doc/source/using.rst b/doc/source/using.rst --- a/doc/source/using.rst +++ b/doc/source/using.rst @@ -28,7 +28,7 @@ *New in version 1.11:* in addition to ``wchar_t``, the C types ``char16_t`` and ``char32_t`` work the same but with a known fixed size. In previous versions, this could be achieved using ``uint16_t`` and -``int32_t`` but without automatic convertion to Python unicodes. +``int32_t`` but without automatic conversion to Python unicodes. Pointers, structures and arrays are more complex: they don't have an obvious Python equivalent. Thus, they correspond to objects of type diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst --- a/doc/source/whatsnew.rst +++ b/doc/source/whatsnew.rst @@ -3,6 +3,23 @@ ====================== +v1.11.3 +======= + +* Fix on CPython 3.x: reading the attributes ``__loader__`` or + ``__spec__`` from the cffi-generated lib modules gave a buggy + SystemError. (These attributes are always None, and provided only to + help compatibility with tools that expect them in all modules.) + +* More Windows fixes: workaround for MSVC not supporting large + literal strings in C code (from + ``ffi.embedding_init_code(large_string)``); and an issue with + ``Py_LIMITED_API`` linking with ``python35.dll/python36.dll`` instead + of ``python3.dll``. + +* Small documentation improvements. + + v1.11.2 ======= diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -186,7 +186,7 @@ `Mailing list `_ """, - version='1.11.2', + version='1.11.3', packages=['cffi'] if cpython else [], package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', '_embedding.h', '_cffi_errors.h']} diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py --- a/testing/cffi1/test_recompiler.py +++ b/testing/cffi1/test_recompiler.py @@ -2270,7 +2270,7 @@ char32_t foo_4bytes(char32_t); """) lib = verify(ffi, "test_char16_char32_type" + no_cpp * "_nocpp", """ - #if !defined(__cplusplus) || __cplusplus < 201103L + #if !defined(__cplusplus) || (!defined(_LIBCPP_VERSION) && __cplusplus < 201103L) typedef uint_least16_t char16_t; typedef uint_least32_t char32_t; #endif @@ -2287,3 +2287,13 @@ def test_char16_char32_plain_c(): test_char16_char32_type(no_cpp=True) + +def test_loader_spec(): + ffi = FFI() + lib = verify(ffi, "test_loader_spec", "") + if sys.version_info < (3,): + assert not hasattr(lib, '__loader__') + assert not hasattr(lib, '__spec__') + else: + assert lib.__loader__ is None + assert lib.__spec__ is None From pypy.commits at gmail.com Thu Jan 11 15:48:55 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 11 Jan 2018 12:48:55 -0800 (PST) Subject: [pypy-commit] cffi release-1.11: md5/sha Message-ID: <5a57cdb7.442d1c0a.fa17.77bb@mx.google.com> Author: Armin Rigo Branch: release-1.11 Changeset: r3069:1aafccb9255d Date: 2018-01-11 21:48 +0100 http://bitbucket.org/cffi/cffi/changeset/1aafccb9255d/ Log: md5/sha diff --git a/doc/source/installation.rst b/doc/source/installation.rst --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -55,11 +55,11 @@ * Checksums of the "source" package version 1.11.3: - - MD5: ... + - MD5: 3d263ec84bb9ad08e0edf2d21312fa1e - - SHA: ... + - SHA: 26a73cc075064cc7cd59ec6c58ca0684d16c4ece - - SHA256: ... + - SHA256: 150708ce9e417858f9a4056b5f364d8c7077fd979b9e35307c2d8a4a8e991fd2 * Or grab the most current version from the `Bitbucket page`_: ``hg clone https://bitbucket.org/cffi/cffi`` From pypy.commits at gmail.com Thu Jan 11 16:40:03 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 11 Jan 2018 13:40:03 -0800 (PST) Subject: [pypy-commit] cffi doc-index: split index and lower headings of old versions so TOC looks nicer Message-ID: <5a57d9b3.4d331c0a.72c89.0302@mx.google.com> Author: Matti Picus Branch: doc-index Changeset: r3071:a8f4e0a70b05 Date: 2018-01-11 23:39 +0200 http://bitbucket.org/cffi/cffi/changeset/a8f4e0a70b05/ Log: split index and lower headings of old versions so TOC looks nicer diff --git a/doc/source/goals.rst b/doc/source/goals.rst new file mode 100644 --- /dev/null +++ b/doc/source/goals.rst @@ -0,0 +1,62 @@ +Goals +----- + +The interface is based on `LuaJIT's FFI`_, and follows a few principles: + +* The goal is to call C code from Python without learning a 3rd language: + existing alternatives require users to learn domain specific language + (Cython_, SWIG_) or API (ctypes_). The CFFI design requires users to know + only C and Python, minimizing the extra bits of API that need to be learned. + +* Keep all the Python-related logic in Python so that you don't need to + write much C code (unlike `CPython native C extensions`_). + +* The preferred way is to work at the level of the API (Application + Programming Interface): the C compiler is called from the declarations + you write to validate and link to the C language constructs. + Alternatively, it is also possible to work at the ABI level + (Application Binary Interface), the way ctypes_ work. + However, on non-Windows platforms, C libraries typically + have a specified C API but not an ABI (e.g. they may + document a "struct" as having at least these fields, but maybe more). + +* Try to be complete. For now some C99 constructs are not supported, + but all C89 should be, including macros (and including macro "abuses", + which you can `manually wrap`_ in saner-looking C functions). + +* Attempt to support both PyPy and CPython, with a reasonable path + for other Python implementations like IronPython and Jython. + +* Note that this project is **not** about embedding executable C code in + Python, unlike `Weave`_. This is about calling existing C libraries + from Python. + +.. _`LuaJIT's FFI`: http://luajit.org/ext_ffi.html +.. _`Cython`: http://www.cython.org +.. _`SWIG`: http://www.swig.org/ +.. _`CPython native C extensions`: http://docs.python.org/extending/extending.html +.. _`native C extensions`: http://docs.python.org/extending/extending.html +.. _`ctypes`: http://docs.python.org/library/ctypes.html +.. _`Weave`: http://wiki.scipy.org/Weave +.. _`manually wrap`: overview.html#abi-versus-api + +Get started by reading `the overview`__. + +.. __: overview.html + + +Comments and bugs +----------------- + +The best way to contact us is on the IRC ``#pypy`` channel of +``irc.freenode.net``. Feel free to discuss matters either there or in +the `mailing list`_. Please report to the `issue tracker`_ any bugs. + +As a general rule, when there is a design issue to resolve, we pick the +solution that is the "most C-like". We hope that this module has got +everything you need to access C code and nothing more. + +--- the authors, Armin Rigo and Maciej Fijalkowski + +.. _`issue tracker`: https://bitbucket.org/cffi/cffi/issues +.. _`mailing list`: https://groups.google.com/forum/#!forum/python-cffi diff --git a/doc/source/index.rst b/doc/source/index.rst --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -6,13 +6,10 @@ code from Python, based on C-like declarations that you can often copy-paste from header files or documentation. -* Goals_ - - * `Comments and bugs`_ - .. toctree:: :maxdepth: 2 + goals whatsnew installation overview @@ -22,65 +19,4 @@ embedding -Goals ------ -The interface is based on `LuaJIT's FFI`_, and follows a few principles: - -* The goal is to call C code from Python without learning a 3rd language: - existing alternatives require users to learn domain specific language - (Cython_, SWIG_) or API (ctypes_). The CFFI design requires users to know - only C and Python, minimizing the extra bits of API that need to be learned. - -* Keep all the Python-related logic in Python so that you don't need to - write much C code (unlike `CPython native C extensions`_). - -* The preferred way is to work at the level of the API (Application - Programming Interface): the C compiler is called from the declarations - you write to validate and link to the C language constructs. - Alternatively, it is also possible to work at the ABI level - (Application Binary Interface), the way ctypes_ work. - However, on non-Windows platforms, C libraries typically - have a specified C API but not an ABI (e.g. they may - document a "struct" as having at least these fields, but maybe more). - -* Try to be complete. For now some C99 constructs are not supported, - but all C89 should be, including macros (and including macro "abuses", - which you can `manually wrap`_ in saner-looking C functions). - -* Attempt to support both PyPy and CPython, with a reasonable path - for other Python implementations like IronPython and Jython. - -* Note that this project is **not** about embedding executable C code in - Python, unlike `Weave`_. This is about calling existing C libraries - from Python. - -.. _`LuaJIT's FFI`: http://luajit.org/ext_ffi.html -.. _`Cython`: http://www.cython.org -.. _`SWIG`: http://www.swig.org/ -.. _`CPython native C extensions`: http://docs.python.org/extending/extending.html -.. _`native C extensions`: http://docs.python.org/extending/extending.html -.. _`ctypes`: http://docs.python.org/library/ctypes.html -.. _`Weave`: http://wiki.scipy.org/Weave -.. _`manually wrap`: overview.html#abi-versus-api - -Get started by reading `the overview`__. - -.. __: overview.html - - -Comments and bugs ------------------ - -The best way to contact us is on the IRC ``#pypy`` channel of -``irc.freenode.net``. Feel free to discuss matters either there or in -the `mailing list`_. Please report to the `issue tracker`_ any bugs. - -As a general rule, when there is a design issue to resolve, we pick the -solution that is the "most C-like". We hope that this module has got -everything you need to access C code and nothing more. - ---- the authors, Armin Rigo and Maciej Fijalkowski - -.. _`issue tracker`: https://bitbucket.org/cffi/cffi/issues -.. _`mailing list`: https://groups.google.com/forum/#!forum/python-cffi diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst --- a/doc/source/whatsnew.rst +++ b/doc/source/whatsnew.rst @@ -95,8 +95,11 @@ .. __: http://bugs.python.org/issue31105 +Older Versions +============== + v1.10.1 -======= +------- (only released inside PyPy 5.8.0) @@ -107,7 +110,7 @@ v1.10 -===== +----- * Issue #295: use calloc() directly instead of PyObject_Malloc()+memset() to handle ffi.new() with a default @@ -169,7 +172,7 @@ v1.9 -==== +---- * Structs with variable-sized arrays as their last field: now we track the length of the array after ``ffi.new()`` is called, just like we @@ -204,7 +207,7 @@ v1.8.3 -====== +------ * When passing a ``void *`` argument to a function with a different pointer type, or vice-versa, the cast occurs automatically, like in C. @@ -217,7 +220,7 @@ v1.8.2 -====== +------ * Issue #283: fixed ``ffi.new()`` on structures/unions with nested anonymous structures/unions, when there is at least one union in @@ -226,7 +229,7 @@ v1.8.1 -====== +------ * CPython 3.x: experimental: the generated C extension modules now use the "limited API", which means that, as a compiled .so/.dll, it should @@ -241,7 +244,7 @@ v1.8 -==== +---- * Removed the restriction that ``ffi.from_buffer()`` cannot be used on byte strings. Now you can get a ``char *`` out of a byte string, @@ -255,7 +258,7 @@ v1.7 -==== +---- * ``ffi.gc(p, None)`` removes the destructor on an object previously created by another call to ``ffi.gc()`` @@ -284,7 +287,7 @@ v1.6 -==== +---- * `ffi.list_types()`_ @@ -307,13 +310,13 @@ v1.5.2 -====== +------ * Fix 1.5.1 for Python 2.6. v1.5.1 -====== +------ * A few installation-time tweaks (thanks Stefano!) @@ -325,7 +328,7 @@ v1.5.0 -====== +------ * Support for `using CFFI for embedding`__. @@ -333,13 +336,13 @@ v1.4.2 -====== +------ Nothing changed from v1.4.1. v1.4.1 -====== +------ * Fix the compilation failure of cffi on CPython 3.5.0. (3.5.1 works; some detail changed that makes some underscore-starting macros @@ -349,7 +352,7 @@ v1.4.0 -====== +------ * A `better way to do callbacks`__ has been added (faster and more portable, and usually cleaner). It is a mechanism for the @@ -390,7 +393,7 @@ v1.3.1 -====== +------ * The optional typedefs (``bool``, ``FILE`` and all Windows types) were not always available from out-of-line FFI objects. @@ -406,7 +409,7 @@ v1.3.0 -====== +------ * Added `ffi.memmove()`_. @@ -441,13 +444,13 @@ v1.2.1 -====== +------ Nothing changed from v1.2.0. v1.2.0 -====== +------ * Out-of-line mode: ``int a[][...];`` can be used to declare a structure field or global variable which is, simultaneously, of total length @@ -500,14 +503,14 @@ v1.1.2 -====== +------ * ``ffi.gc()``: fixed a race condition in multithreaded programs introduced in 1.1.1 v1.1.1 -====== +------ * Out-of-line mode: ``ffi.string()``, ``ffi.buffer()`` and ``ffi.getwinerror()`` didn't accept their arguments as keyword @@ -525,7 +528,7 @@ v1.1.0 -====== +------ * Out-of-line API mode: we can now declare integer types with ``typedef int... foo_t;``. The exact size and signedness of ``foo_t`` @@ -558,13 +561,13 @@ v1.0.3 -====== +------ * Same as 1.0.2, apart from doc and test fixes on some platforms. v1.0.2 -====== +------ * Variadic C functions (ending in a "..." argument) were not supported in the out-of-line ABI mode. This was a bug---there was even a @@ -574,7 +577,7 @@ v1.0.1 -====== +------ * ``ffi.set_source()`` crashed if passed a ``sources=[..]`` argument. Fixed by chrippa on pull request #60. @@ -587,7 +590,7 @@ v1.0.0 -====== +------ * The main news item is out-of-line module generation: From pypy.commits at gmail.com Thu Jan 11 16:46:30 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 11 Jan 2018 13:46:30 -0800 (PST) Subject: [pypy-commit] cffi default: update copyright year Message-ID: <5a57db36.8284df0a.7ed5b.c640@mx.google.com> Author: Matti Picus Branch: Changeset: r3072:ecfa5763b69d Date: 2018-01-11 23:46 +0200 http://bitbucket.org/cffi/cffi/changeset/ecfa5763b69d/ Log: update copyright year diff --git a/doc/source/conf.py b/doc/source/conf.py --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -38,7 +38,7 @@ # General information about the project. project = u'CFFI' -copyright = u'2012-2015, Armin Rigo, Maciej Fijalkowski' +copyright = u'2012-2018, Armin Rigo, Maciej Fijalkowski' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From pypy.commits at gmail.com Thu Jan 11 17:59:07 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 11 Jan 2018 14:59:07 -0800 (PST) Subject: [pypy-commit] pypy default: update to cffi 1.11.3 Message-ID: <5a57ec3b.82641c0a.b3cd.a511@mx.google.com> Author: Armin Rigo Branch: Changeset: r93654:62b31e6e8a3b Date: 2018-01-11 23:58 +0100 http://bitbucket.org/pypy/pypy/changeset/62b31e6e8a3b/ Log: update to cffi 1.11.3 diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi from rpython.rtyper.lltypesystem import rffi -VERSION = "1.11.2" +VERSION = "1.11.3" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.11.2", ("This test_c.py file is for testing a version" +assert __version__ == "1.11.3", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): From pypy.commits at gmail.com Thu Jan 11 17:59:22 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 11 Jan 2018 14:59:22 -0800 (PST) Subject: [pypy-commit] pypy default: update these files too Message-ID: <5a57ec4a.4dafdf0a.af0f0.225c@mx.google.com> Author: Armin Rigo Branch: Changeset: r93655:b601392ff32b Date: 2018-01-11 23:58 +0100 http://bitbucket.org/pypy/pypy/changeset/b601392ff32b/ Log: update these files too diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.11.2" -__version_info__ = (1, 11, 2) +__version__ = "1.11.3" +__version_info__ = (1, 11, 3) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -247,7 +247,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.11.2" + "\ncompiled with cffi version: 1.11.3" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); From pypy.commits at gmail.com Thu Jan 11 18:23:58 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 11 Jan 2018 15:23:58 -0800 (PST) Subject: [pypy-commit] pypy default: forgot this place Message-ID: <5a57f20e.289ddf0a.18d0a.7b79@mx.google.com> Author: Armin Rigo Branch: Changeset: r93656:bc91e92c83f8 Date: 2018-01-12 00:23 +0100 http://bitbucket.org/pypy/pypy/changeset/bc91e92c83f8/ Log: forgot this place diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.11.2 +Version: 1.11.3 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski From pypy.commits at gmail.com Fri Jan 12 02:38:56 2018 From: pypy.commits at gmail.com (Raemi) Date: Thu, 11 Jan 2018 23:38:56 -0800 (PST) Subject: [pypy-commit] extradoc extradoc: register for leysin Message-ID: <5a586610.13811c0a.49683.a343@mx.google.com> Author: Remi Meier Branch: extradoc Changeset: r5863:4c59ca94ccf5 Date: 2018-01-12 08:38 +0100 http://bitbucket.org/pypy/extradoc/changeset/4c59ca94ccf5/ Log: register for leysin diff --git a/sprintinfo/leysin-winter-2018/people.txt b/sprintinfo/leysin-winter-2018/people.txt --- a/sprintinfo/leysin-winter-2018/people.txt +++ b/sprintinfo/leysin-winter-2018/people.txt @@ -12,6 +12,7 @@ Armin Rigo private Matti Picus 17.3/22.3 Ermina Manuel Jacob 17.3/24.3 Ermina +Remi Meier 1[78].3/24.3 Ermina ==================== ============== ======================= **NOTE:** lodging is by default in Ermina. Based on past years, there From pypy.commits at gmail.com Fri Jan 12 02:40:58 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 11 Jan 2018 23:40:58 -0800 (PST) Subject: [pypy-commit] cffi release-1.11: Added tag v1.11.3 for changeset 1aafccb9255d Message-ID: <5a58668a.ec88df0a.571d9.4160@mx.google.com> Author: Armin Rigo Branch: release-1.11 Changeset: r3073:1692c5522360 Date: 2018-01-12 08:39 +0100 http://bitbucket.org/cffi/cffi/changeset/1692c5522360/ Log: Added tag v1.11.3 for changeset 1aafccb9255d diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -14,3 +14,4 @@ 0000000000000000000000000000000000000000 release-0.1 ada126bd7d1e96cc76303c1fca64a556912549d8 v1.11.1 5f9690f5832b0292056df45f72314d69c191f75a v1.11.2 +1aafccb9255dbb36f8e785b65624e39628cee63a v1.11.3 From pypy.commits at gmail.com Fri Jan 12 02:40:59 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 11 Jan 2018 23:40:59 -0800 (PST) Subject: [pypy-commit] cffi default: hg merge release-1.11 Message-ID: <5a58668b.f8b8df0a.7a542.7e65@mx.google.com> Author: Armin Rigo Branch: Changeset: r3074:0ce2c70d5632 Date: 2018-01-12 08:39 +0100 http://bitbucket.org/cffi/cffi/changeset/0ce2c70d5632/ Log: hg merge release-1.11 diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -14,3 +14,4 @@ 0000000000000000000000000000000000000000 release-0.1 ada126bd7d1e96cc76303c1fca64a556912549d8 v1.11.1 5f9690f5832b0292056df45f72314d69c191f75a v1.11.2 +1aafccb9255dbb36f8e785b65624e39628cee63a v1.11.3 From pypy.commits at gmail.com Fri Jan 12 02:41:01 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 11 Jan 2018 23:41:01 -0800 (PST) Subject: [pypy-commit] cffi default: merge heads Message-ID: <5a58668d.e280df0a.2c473.a179@mx.google.com> Author: Armin Rigo Branch: Changeset: r3075:3daecc622902 Date: 2018-01-12 08:40 +0100 http://bitbucket.org/cffi/cffi/changeset/3daecc622902/ Log: merge heads diff --git a/doc/source/conf.py b/doc/source/conf.py --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -38,7 +38,7 @@ # General information about the project. project = u'CFFI' -copyright = u'2012-2015, Armin Rigo, Maciej Fijalkowski' +copyright = u'2012-2018, Armin Rigo, Maciej Fijalkowski' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From pypy.commits at gmail.com Fri Jan 12 03:57:39 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 12 Jan 2018 00:57:39 -0800 (PST) Subject: [pypy-commit] cffi default: Document this PyPy error mode Message-ID: <5a587883.7787df0a.b2546.8427@mx.google.com> Author: Armin Rigo Branch: Changeset: r3076:ee1e69d93c23 Date: 2018-01-12 09:57 +0100 http://bitbucket.org/cffi/cffi/changeset/ee1e69d93c23/ Log: Document this PyPy error mode diff --git a/doc/source/embedding.rst b/doc/source/embedding.rst --- a/doc/source/embedding.rst +++ b/doc/source/embedding.rst @@ -279,12 +279,24 @@ Troubleshooting --------------- -The error message +* The error message cffi extension module 'c_module_name' has unknown version 0x2701 -means that the running Python interpreter located a CFFI version older -than 1.5. CFFI 1.5 or newer must be installed in the running Python. + means that the running Python interpreter located a CFFI version older + than 1.5. CFFI 1.5 or newer must be installed in the running Python. + +* On PyPy, the error message + + debug: pypy_setup_home: directories 'lib-python' and 'lib_pypy' not + found in pypy's shared library location or in any parent directory + + means that the ``libpypy-c.so`` file was found, but the standard library + was not found from this location. This occurs at least on some Linux + distributions, because they put ``libpypy-c.so`` inside ``/usr/lib/``, + instead of the way we recommend, which is: keep that file inside + ``/opt/pypy/bin/`` and put a symlink to there from ``/usr/lib/``. + The quickest fix is to do that change manually. Issues about using the .so From pypy.commits at gmail.com Fri Jan 12 12:21:42 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 12 Jan 2018 09:21:42 -0800 (PST) Subject: [pypy-commit] pypy refactor-slots: hg merge default Message-ID: <5a58eea6.ccc4df0a.f647d.1902@mx.google.com> Author: Ronan Lamy Branch: refactor-slots Changeset: r93657:c4f4a2bf8eac Date: 2018-01-12 17:19 +0000 http://bitbucket.org/pypy/pypy/changeset/c4f4a2bf8eac/ Log: hg merge default diff too long, truncating to 2000 out of 102699 lines diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -71,6 +71,8 @@ ^lib_pypy/.+.c$ ^lib_pypy/.+.o$ ^lib_pypy/.+.so$ +^lib_pypy/.+.pyd$ +^lib_pypy/Release/ ^pypy/doc/discussion/.+\.html$ ^include/.+\.h$ ^include/.+\.inl$ diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -40,3 +40,14 @@ 2875f328eae2216a87f3d6f335092832eb031f56 release-pypy3.5-v5.7.1 c925e73810367cd960a32592dd7f728f436c125c release-pypy2.7-v5.8.0 a37ecfe5f142bc971a86d17305cc5d1d70abec64 release-pypy3.5-v5.8.0 +03d614975835870da65ff0481e1edad68ebbcb8d release-pypy2.7-v5.9.0 +d72f9800a42b46a8056951b1da2426d2c2d8d502 release-pypy3.5-v5.9.0 +03d614975835870da65ff0481e1edad68ebbcb8d release-pypy2.7-v5.9.0 +84a2f3e6a7f88f2fe698e473998755b3bd1a12e2 release-pypy2.7-v5.9.0 +0e7ea4fe15e82d5124e805e2e4a37cae1a402d4b release-pypy2.7-v5.10.0 +a91df6163fb76df245091f741dbf6a23ddc72374 release-pypy3.5-v5.10.0 +a91df6163fb76df245091f741dbf6a23ddc72374 release-pypy3.5-v5.10.0 +0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0 +0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0 +09f9160b643e3f02ccb8c843b2fbb4e5cbf54082 release-pypy3.5-v5.10.0 +3f6eaa010fce78cc7973bdc1dfdb95970f08fed2 release-pypy3.5-v5.10.1 diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -30,7 +30,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2017 +PyPy Copyright holders 2003-2018 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at @@ -339,8 +339,10 @@ Stanisław Halik Julien Phalip Roman Podoliaka + Steve Papanik Eli Stevens Boglarka Vezer + gabrielg PavloKapyshin Tomer Chachamu Christopher Groskopf @@ -363,11 +365,13 @@ Konrad Delong Dinu Gherman pizi + Tomáš Pružina James Robert Armin Ronacher Diana Popa Mads Kiilerich Brett Cannon + Caleb Hattingh aliceinwire Zooko Wilcox-O Hearn James Lan @@ -388,6 +392,7 @@ Jason Madden Yaroslav Fedevych Even Wiik Thomassen + m at funkyhat.org Stefan Marr Heinrich-Heine University, Germany diff --git a/_pytest/terminal.py b/_pytest/terminal.py --- a/_pytest/terminal.py +++ b/_pytest/terminal.py @@ -366,11 +366,11 @@ EXIT_OK, EXIT_TESTSFAILED, EXIT_INTERRUPTED, EXIT_USAGEERROR, EXIT_NOTESTSCOLLECTED) if exitstatus in summary_exit_codes: - self.config.hook.pytest_terminal_summary(terminalreporter=self) self.summary_errors() self.summary_failures() self.summary_warnings() self.summary_passes() + self.config.hook.pytest_terminal_summary(terminalreporter=self) if exitstatus == EXIT_INTERRUPTED: self._report_keyboardinterrupt() del self._keyboardinterrupt_memo diff --git a/extra_tests/requirements.txt b/extra_tests/requirements.txt new file mode 100644 --- /dev/null +++ b/extra_tests/requirements.txt @@ -0,0 +1,3 @@ +pytest +hypothesis +vmprof diff --git a/extra_tests/test_bytes.py b/extra_tests/test_bytes.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_bytes.py @@ -0,0 +1,84 @@ +from hypothesis import strategies as st +from hypothesis import given, example + +st_bytestring = st.binary() | st.binary().map(bytearray) + + at given(st_bytestring, st_bytestring, st_bytestring) +def test_find(u, prefix, suffix): + s = prefix + u + suffix + assert 0 <= s.find(u) <= len(prefix) + assert s.find(u, len(prefix), len(s) - len(suffix)) == len(prefix) + + at given(st_bytestring, st_bytestring, st_bytestring) +def test_index(u, prefix, suffix): + s = prefix + u + suffix + assert 0 <= s.index(u) <= len(prefix) + assert s.index(u, len(prefix), len(s) - len(suffix)) == len(prefix) + + at given(st_bytestring, st_bytestring, st_bytestring) +def test_rfind(u, prefix, suffix): + s = prefix + u + suffix + assert s.rfind(u) >= len(prefix) + assert s.rfind(u, len(prefix), len(s) - len(suffix)) == len(prefix) + + at given(st_bytestring, st_bytestring, st_bytestring) +def test_rindex(u, prefix, suffix): + s = prefix + u + suffix + assert s.rindex(u) >= len(prefix) + assert s.rindex(u, len(prefix), len(s) - len(suffix)) == len(prefix) + +def adjust_indices(u, start, end): + if end < 0: + end = max(end + len(u), 0) + else: + end = min(end, len(u)) + if start < 0: + start = max(start + len(u), 0) + return start, end + + at given(st_bytestring, st_bytestring) +def test_startswith_basic(u, v): + assert u.startswith(v) is (u[:len(v)] == v) + + at example(b'x', b'', 1) + at example(b'x', b'', 2) + at given(st_bytestring, st_bytestring, st.integers()) +def test_startswith_start(u, v, start): + expected = u[start:].startswith(v) if v else (start <= len(u)) + assert u.startswith(v, start) is expected + + at example(b'x', b'', 1, 0) + at example(b'xx', b'', -1, 0) + at given(st_bytestring, st_bytestring, st.integers(), st.integers()) +def test_startswith_3(u, v, start, end): + if v: + expected = u[start:end].startswith(v) + else: # CPython leaks implementation details in this case + start0, end0 = adjust_indices(u, start, end) + expected = start0 <= len(u) and start0 <= end0 + assert u.startswith(v, start, end) is expected + + at given(st_bytestring, st_bytestring) +def test_endswith_basic(u, v): + if len(v) > len(u): + assert u.endswith(v) is False + else: + assert u.endswith(v) is (u[len(u) - len(v):] == v) + + at example(b'x', b'', 1) + at example(b'x', b'', 2) + at given(st_bytestring, st_bytestring, st.integers()) +def test_endswith_2(u, v, start): + expected = u[start:].endswith(v) if v else (start <= len(u)) + assert u.endswith(v, start) is expected + + at example(b'x', b'', 1, 0) + at example(b'xx', b'', -1, 0) + at given(st_bytestring, st_bytestring, st.integers(), st.integers()) +def test_endswith_3(u, v, start, end): + if v: + expected = u[start:end].endswith(v) + else: # CPython leaks implementation details in this case + start0, end0 = adjust_indices(u, start, end) + expected = start0 <= len(u) and start0 <= end0 + assert u.endswith(v, start, end) is expected diff --git a/pypy/module/test_lib_pypy/test_json_extra.py b/extra_tests/test_json.py rename from pypy/module/test_lib_pypy/test_json_extra.py rename to extra_tests/test_json.py --- a/pypy/module/test_lib_pypy/test_json_extra.py +++ b/extra_tests/test_json.py @@ -1,4 +1,6 @@ -import py, json +import pytest +import json +from hypothesis import given, strategies def is_(x, y): return type(x) is type(y) and x == y @@ -6,12 +8,26 @@ def test_no_ensure_ascii(): assert is_(json.dumps(u"\u1234", ensure_ascii=False), u'"\u1234"') assert is_(json.dumps("\xc0", ensure_ascii=False), '"\xc0"') - e = py.test.raises(UnicodeDecodeError, json.dumps, - (u"\u1234", "\xc0"), ensure_ascii=False) - assert str(e.value).startswith("'ascii' codec can't decode byte 0xc0 ") - e = py.test.raises(UnicodeDecodeError, json.dumps, - ("\xc0", u"\u1234"), ensure_ascii=False) - assert str(e.value).startswith("'ascii' codec can't decode byte 0xc0 ") + with pytest.raises(UnicodeDecodeError) as excinfo: + json.dumps((u"\u1234", "\xc0"), ensure_ascii=False) + assert str(excinfo.value).startswith( + "'ascii' codec can't decode byte 0xc0 ") + with pytest.raises(UnicodeDecodeError) as excinfo: + json.dumps(("\xc0", u"\u1234"), ensure_ascii=False) + assert str(excinfo.value).startswith( + "'ascii' codec can't decode byte 0xc0 ") def test_issue2191(): assert is_(json.dumps(u"xxx", ensure_ascii=False), u'"xxx"') + +jsondata = strategies.recursive( + strategies.none() | + strategies.booleans() | + strategies.floats(allow_nan=False) | + strategies.text(), + lambda children: strategies.lists(children) | + strategies.dictionaries(strategies.text(), children)) + + at given(jsondata) +def test_roundtrip(d): + assert json.loads(json.dumps(d)) == d diff --git a/extra_tests/test_textio.py b/extra_tests/test_textio.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_textio.py @@ -0,0 +1,48 @@ +from hypothesis import given, strategies as st + +from io import BytesIO, TextIOWrapper +import os + +def translate_newlines(text): + text = text.replace('\r\n', '\n') + text = text.replace('\r', '\n') + return text.replace('\n', os.linesep) + + at st.composite +def st_readline_universal( + draw, st_nlines=st.integers(min_value=0, max_value=10)): + n_lines = draw(st_nlines) + lines = draw(st.lists( + st.text(st.characters(blacklist_characters='\r\n')), + min_size=n_lines, max_size=n_lines)) + limits = [] + for line in lines: + limit = draw(st.integers(min_value=0, max_value=len(line) + 5)) + limits.append(limit) + limits.append(-1) + endings = draw(st.lists( + st.sampled_from(['\n', '\r', '\r\n']), + min_size=n_lines, max_size=n_lines)) + return ( + ''.join(line + ending for line, ending in zip(lines, endings)), + limits) + + at given(data=st_readline_universal(), + mode=st.sampled_from(['\r', '\n', '\r\n', '', None])) +def test_readline(data, mode): + txt, limits = data + textio = TextIOWrapper( + BytesIO(txt.encode('utf-8', 'surrogatepass')), + encoding='utf-8', errors='surrogatepass', newline=mode) + lines = [] + for limit in limits: + line = textio.readline(limit) + if limit >= 0: + assert len(line) <= limit + if line: + lines.append(line) + elif limit: + break + if mode is None: + txt = translate_newlines(txt) + assert txt.startswith(u''.join(lines)) diff --git a/extra_tests/test_unicode.py b/extra_tests/test_unicode.py --- a/extra_tests/test_unicode.py +++ b/extra_tests/test_unicode.py @@ -1,3 +1,4 @@ +import sys import pytest from hypothesis import strategies as st from hypothesis import given, settings, example @@ -32,3 +33,89 @@ @given(s=st.text()) def test_composition(s, norm1, norm2, norm3): assert normalize(norm2, normalize(norm1, s)) == normalize(norm3, s) + + at given(st.text(), st.text(), st.text()) +def test_find(u, prefix, suffix): + s = prefix + u + suffix + assert 0 <= s.find(u) <= len(prefix) + assert s.find(u, len(prefix), len(s) - len(suffix)) == len(prefix) + + at given(st.text(), st.text(), st.text()) +def test_index(u, prefix, suffix): + s = prefix + u + suffix + assert 0 <= s.index(u) <= len(prefix) + assert s.index(u, len(prefix), len(s) - len(suffix)) == len(prefix) + + at given(st.text(), st.text(), st.text()) +def test_rfind(u, prefix, suffix): + s = prefix + u + suffix + assert s.rfind(u) >= len(prefix) + assert s.rfind(u, len(prefix), len(s) - len(suffix)) == len(prefix) + + at given(st.text(), st.text(), st.text()) +def test_rindex(u, prefix, suffix): + s = prefix + u + suffix + assert s.rindex(u) >= len(prefix) + assert s.rindex(u, len(prefix), len(s) - len(suffix)) == len(prefix) + +def adjust_indices(u, start, end): + if end < 0: + end = max(end + len(u), 0) + else: + end = min(end, len(u)) + if start < 0: + start = max(start + len(u), 0) + return start, end + + at given(st.text(), st.text()) +def test_startswith_basic(u, v): + assert u.startswith(v) is (u[:len(v)] == v) + + at example(u'x', u'', 1) + at example(u'x', u'', 2) + at given(st.text(), st.text(), st.integers()) +def test_startswith_2(u, v, start): + if v or sys.version_info[0] == 2: + expected = u[start:].startswith(v) + else: # CPython leaks implementation details in this case + expected = start <= len(u) + assert u.startswith(v, start) is expected + + at example(u'x', u'', 1, 0) + at example(u'xx', u'', -1, 0) + at given(st.text(), st.text(), st.integers(), st.integers()) +def test_startswith_3(u, v, start, end): + if v or sys.version_info[0] == 2: + expected = u[start:end].startswith(v) + else: # CPython leaks implementation details in this case + start0, end0 = adjust_indices(u, start, end) + expected = start0 <= len(u) and start0 <= end0 + assert u.startswith(v, start, end) is expected + + at given(st.text(), st.text()) +def test_endswith_basic(u, v): + if len(v) > len(u): + assert u.endswith(v) is False + else: + assert u.endswith(v) is (u[len(u) - len(v):] == v) + + at example(u'x', u'', 1) + at example(u'x', u'', 2) + at given(st.text(), st.text(), st.integers()) +def test_endswith_2(u, v, start): + if v or sys.version_info[0] == 2: + expected = u[start:].endswith(v) + else: # CPython leaks implementation details in this case + expected = start <= len(u) + assert u.endswith(v, start) is expected + + at example(u'x', u'', 1, 0) + at example(u'xx', u'', -1, 0) + at given(st.text(), st.text(), st.integers(), st.integers()) +def test_endswith_3(u, v, start, end): + if v or sys.version_info[0] == 2: + expected = u[start:end].endswith(v) + else: # CPython leaks implementation details in this case + start0, end0 = adjust_indices(u, start, end) + expected = start0 <= len(u) and start0 <= end0 + assert u.endswith(v, start, end) is expected diff --git a/extra_tests/test_vmprof_greenlet.py b/extra_tests/test_vmprof_greenlet.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_vmprof_greenlet.py @@ -0,0 +1,28 @@ +import time +import pytest +import greenlet +vmprof = pytest.importorskip('vmprof') + +def count_samples(filename): + stats = vmprof.read_profile(filename) + return len(stats.profiles) + +def cpuburn(duration): + end = time.time() + duration + while time.time() < end: + pass + +def test_sampling_inside_callback(tmpdir): + # see also test_sampling_inside_callback inside + # pypy/module/_continuation/test/test_stacklet.py + # + G = greenlet.greenlet(cpuburn) + fname = tmpdir.join('log.vmprof') + with fname.open('w+b') as f: + vmprof.enable(f.fileno(), 1/250.0) + G.switch(0.1) + vmprof.disable() + + samples = count_samples(str(fname)) + # 0.1 seconds at 250Hz should be 25 samples + assert 23 < samples < 27 diff --git a/lib-python/2.7/ctypes/__init__.py b/lib-python/2.7/ctypes/__init__.py --- a/lib-python/2.7/ctypes/__init__.py +++ b/lib-python/2.7/ctypes/__init__.py @@ -360,14 +360,15 @@ self._FuncPtr = _FuncPtr if handle is None: - if flags & _FUNCFLAG_CDECL: - pypy_dll = _ffi.CDLL(name, mode) - else: - pypy_dll = _ffi.WinDLL(name, mode) - self.__pypy_dll__ = pypy_dll - handle = int(pypy_dll) - if _sys.maxint > 2 ** 32: - handle = int(handle) # long -> int + handle = 0 + if flags & _FUNCFLAG_CDECL: + pypy_dll = _ffi.CDLL(name, mode, handle) + else: + pypy_dll = _ffi.WinDLL(name, mode, handle) + self.__pypy_dll__ = pypy_dll + handle = int(pypy_dll) + if _sys.maxint > 2 ** 32: + handle = int(handle) # long -> int self._handle = handle def __repr__(self): diff --git a/lib-python/2.7/inspect.py b/lib-python/2.7/inspect.py --- a/lib-python/2.7/inspect.py +++ b/lib-python/2.7/inspect.py @@ -40,6 +40,10 @@ import linecache from operator import attrgetter from collections import namedtuple +try: + from cpyext import is_cpyext_function as _is_cpyext_function +except ImportError: + _is_cpyext_function = lambda obj: False # These constants are from Include/code.h. CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS = 0x1, 0x2, 0x4, 0x8 @@ -230,7 +234,7 @@ __doc__ documentation string __name__ original name of this function or method __self__ instance to which a method is bound, or None""" - return isinstance(object, types.BuiltinFunctionType) + return isinstance(object, types.BuiltinFunctionType) or _is_cpyext_function(object) def isroutine(object): """Return true if the object is any kind of function or method.""" diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py --- a/lib-python/2.7/subprocess.py +++ b/lib-python/2.7/subprocess.py @@ -1296,7 +1296,7 @@ 'copyfile' in caller.f_globals): dest_dir = sys.pypy_resolvedirof(target_executable) src_dir = sys.pypy_resolvedirof(sys.executable) - for libname in ['libpypy-c.so', 'libpypy-c.dylib']: + for libname in ['libpypy-c.so', 'libpypy-c.dylib', 'libpypy-c.dll']: dest_library = os.path.join(dest_dir, libname) src_library = os.path.join(src_dir, libname) if os.path.exists(src_library): diff --git a/lib-python/2.7/test/test_urllib2net.py b/lib-python/2.7/test/test_urllib2net.py --- a/lib-python/2.7/test/test_urllib2net.py +++ b/lib-python/2.7/test/test_urllib2net.py @@ -286,7 +286,7 @@ self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 120) u.close() - FTP_HOST = 'ftp://ftp.debian.org/debian/' + FTP_HOST = 'ftp://www.pythontest.net/' def test_ftp_basic(self): self.assertIsNone(socket.getdefaulttimeout()) diff --git a/lib-python/2.7/warnings.py b/lib-python/2.7/warnings.py --- a/lib-python/2.7/warnings.py +++ b/lib-python/2.7/warnings.py @@ -43,11 +43,12 @@ unicodetype = unicode except NameError: unicodetype = () + template = "%s: %s: %s\n" try: message = str(message) except UnicodeEncodeError: - pass - s = "%s: %s: %s\n" % (lineno, category.__name__, message) + template = unicode(template) + s = template % (lineno, category.__name__, message) line = linecache.getline(filename, lineno) if line is None else line if line: line = line.strip() diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py --- a/lib_pypy/_ctypes/array.py +++ b/lib_pypy/_ctypes/array.py @@ -8,60 +8,64 @@ class ArrayMeta(_CDataMeta): def __new__(self, name, cls, typedict): res = type.__new__(self, name, cls, typedict) - if '_type_' in typedict: - ffiarray = _rawffi.Array(typedict['_type_']._ffishape_) - res._ffiarray = ffiarray - subletter = getattr(typedict['_type_'], '_type_', None) - if subletter == 'c': - def getvalue(self): - return _rawffi.charp2string(self._buffer.buffer, - self._length_) - def setvalue(self, val): - # we don't want to have buffers here - if len(val) > self._length_: - raise ValueError("%r too long" % (val,)) - if isinstance(val, str): - _rawffi.rawstring2charp(self._buffer.buffer, val) - else: - for i in range(len(val)): - self[i] = val[i] - if len(val) < self._length_: - self._buffer[len(val)] = '\x00' - res.value = property(getvalue, setvalue) - def getraw(self): - return _rawffi.charp2rawstring(self._buffer.buffer, - self._length_) + if cls == (_CData,): # this is the Array class defined below + res._ffiarray = None + return res + if not hasattr(res, '_length_') or not isinstance(res._length_, + (int, long)): + raise AttributeError( + "class must define a '_length_' attribute, " + "which must be a positive integer") + ffiarray = res._ffiarray = _rawffi.Array(res._type_._ffishape_) + subletter = getattr(res._type_, '_type_', None) + if subletter == 'c': + def getvalue(self): + return _rawffi.charp2string(self._buffer.buffer, + self._length_) + def setvalue(self, val): + # we don't want to have buffers here + if len(val) > self._length_: + raise ValueError("%r too long" % (val,)) + if isinstance(val, str): + _rawffi.rawstring2charp(self._buffer.buffer, val) + else: + for i in range(len(val)): + self[i] = val[i] + if len(val) < self._length_: + self._buffer[len(val)] = b'\x00' + res.value = property(getvalue, setvalue) - def setraw(self, buffer): - if len(buffer) > self._length_: - raise ValueError("%r too long" % (buffer,)) - _rawffi.rawstring2charp(self._buffer.buffer, buffer) - res.raw = property(getraw, setraw) - elif subletter == 'u': - def getvalue(self): - return _rawffi.wcharp2unicode(self._buffer.buffer, - self._length_) + def getraw(self): + return _rawffi.charp2rawstring(self._buffer.buffer, + self._length_) - def setvalue(self, val): - # we don't want to have buffers here - if len(val) > self._length_: - raise ValueError("%r too long" % (val,)) - if isinstance(val, unicode): - target = self._buffer - else: - target = self - for i in range(len(val)): - target[i] = val[i] - if len(val) < self._length_: - target[len(val)] = u'\x00' - res.value = property(getvalue, setvalue) - - if '_length_' in typedict: - res._ffishape_ = (ffiarray, typedict['_length_']) - res._fficompositesize_ = res._sizeofinstances() - else: - res._ffiarray = None + def setraw(self, buffer): + if len(buffer) > self._length_: + raise ValueError("%r too long" % (buffer,)) + _rawffi.rawstring2charp(self._buffer.buffer, buffer) + res.raw = property(getraw, setraw) + elif subletter == 'u': + def getvalue(self): + return _rawffi.wcharp2unicode(self._buffer.buffer, + self._length_) + + def setvalue(self, val): + # we don't want to have buffers here + if len(val) > self._length_: + raise ValueError("%r too long" % (val,)) + if isinstance(val, unicode): + target = self._buffer + else: + target = self + for i in range(len(val)): + target[i] = val[i] + if len(val) < self._length_: + target[len(val)] = u'\x00' + res.value = property(getvalue, setvalue) + + res._ffishape_ = (ffiarray, res._length_) + res._fficompositesize_ = res._sizeofinstances() return res from_address = cdata_from_address @@ -156,7 +160,7 @@ l = [self[i] for i in range(start, stop, step)] letter = getattr(self._type_, '_type_', None) if letter == 'c': - return "".join(l) + return b"".join(l) if letter == 'u': return u"".join(l) return l diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py --- a/lib_pypy/_ctypes/basics.py +++ b/lib_pypy/_ctypes/basics.py @@ -176,6 +176,10 @@ def _get_buffer_value(self): return self._buffer[0] + def _copy_to(self, addr): + target = type(self).from_address(addr)._buffer + target[0] = self._get_buffer_value() + def _to_ffi_param(self): if self.__class__._is_pointer_like(): return self._get_buffer_value() diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py --- a/lib_pypy/_ctypes/pointer.py +++ b/lib_pypy/_ctypes/pointer.py @@ -114,7 +114,9 @@ cobj = self._type_.from_param(value) if ensure_objects(cobj) is not None: store_reference(self, index, cobj._objects) - self._subarray(index)[0] = cobj._get_buffer_value() + address = self._buffer[0] + address += index * sizeof(self._type_) + cobj._copy_to(address) def __nonzero__(self): return self._buffer[0] != 0 diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -291,6 +291,11 @@ def _get_buffer_value(self): return self._buffer.buffer + def _copy_to(self, addr): + from ctypes import memmove + origin = self._get_buffer_value() + memmove(addr, origin, self._fficompositesize_) + def _to_ffi_param(self): return self._buffer diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py --- a/lib_pypy/_ctypes_test.py +++ b/lib_pypy/_ctypes_test.py @@ -21,5 +21,11 @@ with fp: imp.load_module('_ctypes_test', fp, filename, description) except ImportError: + if os.name == 'nt': + # hack around finding compilers on win32 + try: + import setuptools + except ImportError: + pass print('could not find _ctypes_test in %s' % output_dir) _pypy_testcapi.compile_shared('_ctypes_test.c', '_ctypes_test', output_dir) diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -1027,21 +1027,25 @@ if '\0' in sql: raise ValueError("the query contains a null character") - first_word = sql.lstrip().split(" ")[0].upper() - if first_word == "": + + if sql: + first_word = sql.lstrip().split()[0].upper() + if first_word == '': + self._type = _STMT_TYPE_INVALID + if first_word == "SELECT": + self._type = _STMT_TYPE_SELECT + elif first_word == "INSERT": + self._type = _STMT_TYPE_INSERT + elif first_word == "UPDATE": + self._type = _STMT_TYPE_UPDATE + elif first_word == "DELETE": + self._type = _STMT_TYPE_DELETE + elif first_word == "REPLACE": + self._type = _STMT_TYPE_REPLACE + else: + self._type = _STMT_TYPE_OTHER + else: self._type = _STMT_TYPE_INVALID - elif first_word == "SELECT": - self._type = _STMT_TYPE_SELECT - elif first_word == "INSERT": - self._type = _STMT_TYPE_INSERT - elif first_word == "UPDATE": - self._type = _STMT_TYPE_UPDATE - elif first_word == "DELETE": - self._type = _STMT_TYPE_DELETE - elif first_word == "REPLACE": - self._type = _STMT_TYPE_REPLACE - else: - self._type = _STMT_TYPE_OTHER if isinstance(sql, unicode): sql = sql.encode('utf-8') diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py --- a/lib_pypy/_testcapi.py +++ b/lib_pypy/_testcapi.py @@ -16,4 +16,10 @@ with fp: imp.load_module('_testcapi', fp, filename, description) except ImportError: + if os.name == 'nt': + # hack around finding compilers on win32 + try: + import setuptools + except ImportError: + pass _pypy_testcapi.compile_shared(cfile, '_testcapi', output_dir) diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py --- a/lib_pypy/_tkinter/app.py +++ b/lib_pypy/_tkinter/app.py @@ -119,7 +119,7 @@ tklib.TCL_GLOBAL_ONLY) # This is used to get the application class for Tk 4.1 and up - argv0 = className.lower() + argv0 = className.lower().encode('ascii') tklib.Tcl_SetVar(self.interp, "argv0", argv0, tklib.TCL_GLOBAL_ONLY) @@ -180,6 +180,9 @@ if err == tklib.TCL_ERROR: self.raiseTclError() + def interpaddr(self): + return int(tkffi.cast('size_t', self.interp)) + def _var_invoke(self, func, *args, **kwargs): if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread(): # The current thread is not the interpreter thread. diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.11.1 +Version: 1.11.3 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.11.1" -__version_info__ = (1, 11, 1) +__version__ = "1.11.3" +__version_info__ = (1, 11, 3) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h --- a/lib_pypy/cffi/_cffi_include.h +++ b/lib_pypy/cffi/_cffi_include.h @@ -7,11 +7,38 @@ we can learn about Py_DEBUG from pyconfig.h, but it is unclear if the same works for the other two macros. Py_DEBUG implies them, but not the other way around. + + Issue #350: more mess: on Windows, with _MSC_VER, we have to define + Py_LIMITED_API even before including pyconfig.h. In that case, we + guess what pyconfig.h will do to the macros above, and check our + guess after the #include. */ #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) -# include -# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) -# define Py_LIMITED_API +# ifdef _MSC_VER +# if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API +# endif +# include + /* sanity-check: Py_LIMITED_API will cause crashes if any of these + are also defined. Normally, the Python file PC/pyconfig.h does not + cause any of these to be defined, with the exception that _DEBUG + causes Py_DEBUG. Double-check that. */ +# ifdef Py_LIMITED_API +# if defined(Py_DEBUG) +# error "pyconfig.h unexpectedly defines Py_DEBUG but _DEBUG is not set" +# endif +# if defined(Py_TRACE_REFS) +# error "pyconfig.h unexpectedly defines Py_TRACE_REFS" +# endif +# if defined(Py_REF_DEBUG) +# error "pyconfig.h unexpectedly defines Py_REF_DEBUG" +# endif +# endif +# else +# include +# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API +# endif # endif #endif @@ -238,9 +265,9 @@ _CFFI_UNUSED_FN static PyObject *_cffi_from_c_char16_t(uint16_t x) { if (sizeof(_cffi_wchar_t) == 2) - return _cffi_from_c_wchar_t(x); + return _cffi_from_c_wchar_t((_cffi_wchar_t)x); else - return _cffi_from_c_wchar3216_t(x); + return _cffi_from_c_wchar3216_t((int)x); } _CFFI_UNUSED_FN static int _cffi_to_c_char32_t(PyObject *o) @@ -254,7 +281,7 @@ _CFFI_UNUSED_FN static PyObject *_cffi_from_c_char32_t(int x) { if (sizeof(_cffi_wchar_t) == 4) - return _cffi_from_c_wchar_t(x); + return _cffi_from_c_wchar_t((_cffi_wchar_t)x); else return _cffi_from_c_wchar3216_t(x); } diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -247,7 +247,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.11.1" + "\ncompiled with cffi version: 1.11.3" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -295,8 +295,9 @@ base_module_name = self.module_name.split('.')[-1] if self.ffi._embedding is not None: prnt('#define _CFFI_MODULE_NAME "%s"' % (self.module_name,)) - prnt('#define _CFFI_PYTHON_STARTUP_CODE %s' % - (self._string_literal(self.ffi._embedding),)) + prnt('static const char _CFFI_PYTHON_STARTUP_CODE[] = {') + self._print_string_literal_in_array(self.ffi._embedding) + prnt('0 };') prnt('#ifdef PYPY_VERSION') prnt('# define _CFFI_PYTHON_STARTUP_FUNC _cffi_pypyinit_%s' % ( base_module_name,)) @@ -1271,17 +1272,18 @@ _generate_cpy_extern_python_plus_c_ctx = \ _generate_cpy_extern_python_ctx - def _string_literal(self, s): - def _char_repr(c): - # escape with a '\' the characters '\', '"' or (for trigraphs) '?' - if c in '\\"?': return '\\' + c - if ' ' <= c < '\x7F': return c - if c == '\n': return '\\n' - return '\\%03o' % ord(c) - lines = [] - for line in s.splitlines(True) or ['']: - lines.append('"%s"' % ''.join([_char_repr(c) for c in line])) - return ' \\\n'.join(lines) + def _print_string_literal_in_array(self, s): + prnt = self._prnt + prnt('// # NB. this is not a string because of a size limit in MSVC') + for line in s.splitlines(True): + prnt(('// ' + line).rstrip()) + printed_line = '' + for c in line: + if len(printed_line) >= 76: + prnt(printed_line) + printed_line = '' + printed_line += '%d,' % (ord(c),) + prnt(printed_line) # ---------- # emitting the opcodes for individual types diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py --- a/lib_pypy/cffi/verifier.py +++ b/lib_pypy/cffi/verifier.py @@ -301,7 +301,6 @@ return suffixes def _ensure_dir(filename): - try: - os.makedirs(os.path.dirname(filename)) - except OSError: - pass + dirname = os.path.dirname(filename) + if dirname and not os.path.isdir(dirname): + os.makedirs(dirname) diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py --- a/lib_pypy/resource.py +++ b/lib_pypy/resource.py @@ -20,6 +20,7 @@ or via the attributes ru_utime, ru_stime, ru_maxrss, and so on.""" __metaclass__ = _structseq.structseqtype + name = "resource.struct_rusage" ru_utime = _structseq.structseqfield(0, "user time used") ru_stime = _structseq.structseqfield(1, "system time used") diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst --- a/pypy/doc/build.rst +++ b/pypy/doc/build.rst @@ -119,7 +119,7 @@ To run untranslated tests, you need the Boehm garbage collector libgc. -On recent Debian and Ubuntu (like 17.04), this is the command to install +On recent Debian and Ubuntu (16.04 onwards), this is the command to install all build-time dependencies:: apt-get install gcc make libffi-dev pkg-config zlib1g-dev libbz2-dev \ @@ -127,7 +127,7 @@ tk-dev libgc-dev python-cffi \ liblzma-dev libncursesw5-dev # these two only needed on PyPy3 -On older Debian and Ubuntu (12.04 to 16.04):: +On older Debian and Ubuntu (12.04-14.04):: apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \ libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \ @@ -149,12 +149,23 @@ xz-devel # For lzma on PyPy3. (XXX plus the SLES11 version of libgdbm-dev and tk-dev) -On Mac OS X, most of these build-time dependencies are installed alongside +On Mac OS X: + +Most of these build-time dependencies are installed alongside the Developer Tools. However, note that in order for the installation to find them you may need to run:: xcode-select --install +An exception is OpenSSL, which is no longer provided with the operating +system. It can be obtained via Homebrew (with ``$ brew install openssl``), +but it will not be available on the system path by default. The easiest +way to enable it for building pypy is to set an environment variable:: + + export PKG_CONFIG_PATH=$(brew --prefix)/opt/openssl/lib/pkgconfig + +After setting this, translation (described next) will find the OpenSSL libs +as expected. Run the translation ------------------- @@ -187,18 +198,18 @@ entire pypy interpreter. This step is currently singe threaded, and RAM hungry. As part of this step, the chain creates a large number of C code files and a Makefile to compile them in a - directory controlled by the ``PYPY_USESSION_DIR`` environment variable. + directory controlled by the ``PYPY_USESSION_DIR`` environment variable. 2. Create an executable ``pypy-c`` by running the Makefile. This step can - utilize all possible cores on the machine. -3. Copy the needed binaries to the current directory. -4. Generate c-extension modules for any cffi-based stdlib modules. + utilize all possible cores on the machine. +3. Copy the needed binaries to the current directory. +4. Generate c-extension modules for any cffi-based stdlib modules. The resulting executable behaves mostly like a normal Python interpreter (see :doc:`cpython_differences`), and is ready for testing, for use as a base interpreter for a new virtualenv, or for packaging into a binary suitable for installation on another machine running the same OS as the build -machine. +machine. Note that step 4 is merely done as a convenience, any of the steps may be rerun without rerunning the previous steps. @@ -255,7 +266,7 @@ * PyPy 2.5.1 or earlier: normal users would see permission errors. Installers need to run ``pypy -c "import gdbm"`` and other similar - commands at install time; the exact list is in + commands at install time; the exact list is in :source:`pypy/tool/release/package.py `. Users seeing a broken installation of PyPy can fix it after-the-fact if they have sudo rights, by running once e.g. ``sudo pypy -c "import gdbm``. diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -59,7 +59,7 @@ # General information about the project. project = u'PyPy' -copyright = u'2017, The PyPy Project' +copyright = u'2018, The PyPy Project' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -217,6 +217,7 @@ Alejandro J. Cura Vladimir Kryachko Gabriel + Thomas Hisch Mark Williams Kunal Grover Nathan Taylor @@ -306,8 +307,10 @@ Stanisław Halik Julien Phalip Roman Podoliaka + Steve Papanik Eli Stevens Boglarka Vezer + gabrielg PavloKapyshin Tomer Chachamu Christopher Groskopf @@ -330,11 +333,13 @@ Konrad Delong Dinu Gherman pizi + Tomáš Pružina James Robert Armin Ronacher Diana Popa Mads Kiilerich Brett Cannon + Caleb Hattingh aliceinwire Zooko Wilcox-O Hearn James Lan @@ -355,4 +360,5 @@ Jason Madden Yaroslav Fedevych Even Wiik Thomassen + m at funkyhat.org Stefan Marr diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -355,7 +355,11 @@ containers (as list items or in sets for example), the exact rule of equality used is "``if x is y or x == y``" (on both CPython and PyPy); as a consequence, because all ``nans`` are identical in PyPy, you -cannot have several of them in a set, unlike in CPython. (Issue `#1974`__) +cannot have several of them in a set, unlike in CPython. (Issue `#1974`__). +Another consequence is that ``cmp(float('nan'), float('nan')) == 0``, because +``cmp`` checks with ``is`` first whether the arguments are identical (there is +no good value to return from this call to ``cmp``, because ``cmp`` pretends +that there is a total order on floats, but that is wrong for NaNs). .. __: https://bitbucket.org/pypy/pypy/issue/1974/different-behaviour-for-collections-of @@ -541,6 +545,15 @@ ``del foo.bar`` where ``foo`` is a module (or class) that contains the function ``bar``, is significantly slower than CPython. +* Various built-in functions in CPython accept only positional arguments + and not keyword arguments. That can be considered a long-running + historical detail: newer functions tend to accept keyword arguments + and older function are occasionally fixed to do so as well. In PyPy, + most built-in functions accept keyword arguments (``help()`` shows the + argument names). But don't rely on it too much because future + versions of PyPy may have to rename the arguments if CPython starts + accepting them too. + .. _`is ignored in PyPy`: http://bugs.python.org/issue14621 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/ diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst --- a/pypy/doc/faq.rst +++ b/pypy/doc/faq.rst @@ -182,6 +182,57 @@ technical difficulties. +What about numpy, numpypy, micronumpy? +-------------------------------------- + +Way back in 2011, the PyPy team `started to reimplement`_ numpy in PyPy. It +has two pieces: + + * the builtin module :source:`pypy/module/micronumpy`: this is written in + RPython and roughly covers the content of the ``numpy.core.multiarray`` + module. Confusingly enough, this is available in PyPy under the name + ``_numpypy``. It is included by default in all the official releases of + PyPy (but it might be dropped in the future). + + * a fork_ of the official numpy repository maintained by us and informally + called ``numpypy``: even more confusing, the name of the repo on bitbucket + is ``numpy``. The main difference with the upstream numpy, is that it is + based on the micronumpy module written in RPython, instead of of + ``numpy.core.multiarray`` which is written in C. + +Moreover, it is also possible to install the upstream version of ``numpy``: +its core is written in C and it runs on PyPy under the cpyext compatibility +layer. This is what you get if you do ``pypy -m pip install numpy``. + + +Should I install numpy or numpypy? +----------------------------------- + +TL;DR version: you should use numpy. You can install it by doing ``pypy -m pip +install numpy``. You might also be interested in using the experimental `PyPy +binary wheels`_ to save compilation time. + +The upstream ``numpy`` is written in C, and runs under the cpyext +compatibility layer. Nowadays, cpyext is mature enough that you can simply +use the upstream ``numpy``, since it passes 99.9% of the test suite. At the +moment of writing (October 2017) the main drawback of ``numpy`` is that cpyext +is infamously slow, and thus it has worse performance compared to +``numpypy``. However, we are actively working on improving it, as we expect to +reach the same speed, eventually. + +On the other hand, ``numpypy`` is more JIT-friendly and very fast to call, +since it is written in RPython: but it is a reimplementation, and it's hard to +be completely compatible: over the years the project slowly matured and +eventually it was able to call out to the LAPACK and BLAS libraries to speed +matrix calculations, and reached around an 80% parity with the upstream +numpy. However, 80% is far from 100%. Since cpyext/numpy compatibility is +progressing fast, we have discontinued support for ``numpypy``. + +.. _`started to reimplement`: https://morepypy.blogspot.co.il/2011/05/numpy-in-pypy-status-and-roadmap.html +.. _fork: https://bitbucket.org/pypy/numpy +.. _`PyPy binary wheels`: https://github.com/antocuni/pypy-wheels + + Is PyPy more clever than CPython about Tail Calls? -------------------------------------------------- diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst --- a/pypy/doc/how-to-release.rst +++ b/pypy/doc/how-to-release.rst @@ -62,7 +62,7 @@ * go to pypy/tool/release and run ``force-builds.py `` The following JIT binaries should be built, however, we need more buildbots - windows, linux-32, linux-64, osx64, armhf-raring, armhf-raspberrian, armel, + windows, linux-32, linux-64, osx64, armhf-raspberrian, armel, freebsd64 * wait for builds to complete, make sure there are no failures diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,8 @@ .. toctree:: + release-v5.10.1.rst + release-v5.10.0.rst release-v5.9.0.rst release-v5.8.0.rst release-v5.7.1.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ .. toctree:: whatsnew-head.rst + whatsnew-pypy2-5.10.0.rst whatsnew-pypy2-5.9.0.rst whatsnew-pypy2-5.8.0.rst whatsnew-pypy2-5.7.0.rst diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -240,9 +240,12 @@ **matplotlib** https://github.com/matplotlib/matplotlib - TODO: the tkagg backend does not work, which makes tests fail on downstream - projects like Pandas, SciPy. It uses id(obj) as a c-pointer to obj in - tkagg.py, which requires refactoring + Status: using the matplotlib branch of PyPy and the tkagg-cffi branch of + matplotlib from https://github.com/mattip/matplotlib/tree/tkagg-cffi, the + tkagg backend can function. + + TODO: the matplotlib branch passes numpy arrays by value (copying all the + data), this proof-of-concept needs help to become completely compliant **wxPython** https://bitbucket.org/amauryfa/wxpython-cffi diff --git a/pypy/doc/release-v5.10.0.rst b/pypy/doc/release-v5.10.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v5.10.0.rst @@ -0,0 +1,100 @@ +====================================== +PyPy2.7 and PyPy3.5 v5.10 dual release +====================================== + +The PyPy team is proud to release both PyPy2.7 v5.10 (an interpreter supporting +Python 2.7 syntax), and a final PyPy3.5 v5.10 (an interpreter for Python +3.5 syntax). The two releases are both based on much the same codebase, thus +the dual release. + +This release is an incremental release with very few new features, the main +feature being the final PyPy3.5 release that works on linux and OS X with beta +windows support. It also includes fixes for `vmprof`_ cooperation with greenlets. + +Compared to 5.9, the 5.10 release contains mostly bugfixes and small improvements. +We have in the pipeline big new features coming for PyPy 6.0 that did not make +the release cut and should be available within the next couple months. + +As always, this release is 100% compatible with the previous one and fixed +several issues and bugs raised by the growing community of PyPy users. +As always, we strongly recommend updating. + +There are quite a few important changes that are in the pipeline that did not +make it into the 5.10 release. Most important are speed improvements to cpyext +(which will make numpy and pandas a bit faster) and utf8 branch that changes +internal representation of unicode to utf8, which should help especially the +Python 3.5 version of PyPy. + +This release concludes the Mozilla Open Source `grant`_ for having a compatible +PyPy 3.5 release and we're very grateful for that. Of course, we will continue +to improve PyPy 3.5 and probably move to 3.6 during the course of 2018. + +You can download the v5.10 releases here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. + +We would also like to thank our contributors and +encourage new people to join the project. PyPy has many +layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation +improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ +with making RPython's JIT even better. + +.. _vmprof: http://vmprof.readthedocs.io +.. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html +.. _`PyPy`: index.html +.. _`RPython`: https://rpython.readthedocs.org +.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: project-ideas.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +The PyPy release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + +Changelog +========= + +* improve ssl handling on windows for pypy3 (makes pip work) +* improve unicode handling in various error reporters +* fix vmprof cooperation with greenlets +* fix some things in cpyext +* test and document the cmp(nan, nan) == 0 behaviour +* don't crash when calling sleep with inf or nan +* fix bugs in _io module +* inspect.isbuiltin() now returns True for functions implemented in C +* allow the sequences future-import, docstring, future-import for CPython bug compatibility +* Issue #2699: non-ascii messages in warnings +* posix.lockf +* fixes for FreeBSD platform +* add .debug files, so builds contain debugging info, instead of being stripped +* improvements to cppyy +* issue #2677 copy pure c PyBuffer_{From,To}Contiguous from cpython +* issue #2682, split firstword on any whitespace in sqlite3 +* ctypes: allow ptr[0] = foo when ptr is a pointer to struct +* matplotlib will work with tkagg backend once `matplotlib pr #9356`_ is merged +* improvements to utf32 surrogate handling +* cffi version bump to 1.11.2 + +.. _`matplotlib pr #9356`: https://github.com/matplotlib/matplotlib/pull/9356 diff --git a/pypy/doc/release-v5.10.1.rst b/pypy/doc/release-v5.10.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v5.10.1.rst @@ -0,0 +1,63 @@ +=========== +PyPy 5.10.1 +=========== + +We have released a bugfix PyPy3.5-v5.10.1 +due to the following issues: + + * Fix ``time.sleep(float('nan')`` which would hang on windows + + * Fix missing ``errno`` constants on windows + + * Fix issue 2718_ for the REPL on linux + + * Fix an overflow in converting 3 secs to nanosecs (issue 2717_ ) + + * Flag kwarg to ``os.setxattr`` had no effect + + * Fix the winreg module for unicode entries in the registry on windows + +Note that many of these fixes are for our new beta verison of PyPy3.5 on +windows. There may be more unicode problems in the windows beta version +especially around the subject of directory- and file-names with non-ascii +characters. + +Our downloads are available now. On macos, we recommend you wait for the +Homebrew_ package. + +Thanks to those who reported the issues. + +.. _2718: https://bitbucket.org/pypy/pypy/issues/2718 +.. _2717: https://bitbucket.org/pypy/pypy/issues/2717 +.. _Homebrew: http://brewformulas.org/Pypy + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +This PyPy 3.5 release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + +Please update, and continue to help us make PyPy better. + +Cheers + +The PyPy Team + diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py --- a/pypy/doc/tool/makecontributor.py +++ b/pypy/doc/tool/makecontributor.py @@ -81,6 +81,7 @@ 'Yasir Suhail':['yasirs'], 'Squeaky': ['squeaky'], "Amaury Forgeot d'Arc": ['amauryfa at gmail.com'], + "Dodan Mihai": ['mihai.dodan at gmail.com'], } alias_map = {} diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,15 +1,12 @@ -=========================== -What's new in PyPy2.7 5.10+ -=========================== - -.. this is a revision shortly after release-pypy2.7-v5.9.0 -.. startrev:899e5245de1e - -.. branch: cpyext-jit - -Differentiate the code to call METH_NOARGS, METH_O and METH_VARARGS in cpyext: -this allows to write specialized code which is much faster than previous -completely generic version. Moreover, let the JIT to look inside the cpyext -module: the net result is that cpyext calls are up to 7x faster. However, this -is true only for very simple situations: in all real life code, we are still -much slower than CPython (more optimizations to come) +=========================== +What's new in PyPy2.7 5.10+ +=========================== + +.. this is a revision shortly after release-pypy2.7-v5.10.0 +.. startrev: 6b024edd9d12 + +.. branch: cpyext-avoid-roundtrip + +Big refactoring of some cpyext code, which avoids a lot of nonsense when +calling C from Python and vice-versa: the result is a big speedup in +function/method calls, up to 6 times faster. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-5.10.0.rst copy from pypy/doc/whatsnew-head.rst copy to pypy/doc/whatsnew-pypy2-5.10.0.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-pypy2-5.10.0.rst @@ -1,15 +1,42 @@ -=========================== -What's new in PyPy2.7 5.10+ -=========================== - -.. this is a revision shortly after release-pypy2.7-v5.9.0 -.. startrev:899e5245de1e - -.. branch: cpyext-jit - -Differentiate the code to call METH_NOARGS, METH_O and METH_VARARGS in cpyext: -this allows to write specialized code which is much faster than previous -completely generic version. Moreover, let the JIT to look inside the cpyext -module: the net result is that cpyext calls are up to 7x faster. However, this -is true only for very simple situations: in all real life code, we are still -much slower than CPython (more optimizations to come) +========================== +What's new in PyPy2.7 5.10 +========================== + +.. this is a revision shortly after release-pypy2.7-v5.9.0 +.. startrev:d56dadcef996 + + +.. branch: cppyy-packaging + +Cleanup and improve cppyy packaging + +.. branch: docs-osx-brew-openssl + +.. branch: keep-debug-symbols + +Add a smartstrip tool, which can optionally keep the debug symbols in a +separate file, instead of just stripping them away. Use it in packaging + +.. branch: bsd-patches + +Fix failures on FreeBSD, contributed by David Naylor as patches on the issue +tracker (issues 2694, 2695, 2696, 2697) + +.. branch: run-extra-tests + +Run extra_tests/ in buildbot + +.. branch: vmprof-0.4.10 + +Upgrade the _vmprof backend to vmprof 0.4.10 + +.. branch: fix-vmprof-stacklet-switch +.. branch: fix-vmprof-stacklet-switch-2 +Fix a vmprof+continulets (i.e. greenelts, eventlet, gevent, ...) + +.. branch: win32-vcvars + +.. branch: rdict-fast-hash + +Make it possible to declare that the hash function of an r_dict is fast in RPython. + diff --git a/pypy/doc/whatsnew-pypy2-5.6.0.rst b/pypy/doc/whatsnew-pypy2-5.6.0.rst --- a/pypy/doc/whatsnew-pypy2-5.6.0.rst +++ b/pypy/doc/whatsnew-pypy2-5.6.0.rst @@ -101,7 +101,7 @@ .. branch: newinitwarn -Match CPython's stricter handling of __new/init__ arguments +Match CPython's stricter handling of ``__new__``/``__init__`` arguments .. branch: openssl-1.1 diff --git a/pypy/doc/whatsnew-pypy2-5.9.0.rst b/pypy/doc/whatsnew-pypy2-5.9.0.rst --- a/pypy/doc/whatsnew-pypy2-5.9.0.rst +++ b/pypy/doc/whatsnew-pypy2-5.9.0.rst @@ -85,3 +85,12 @@ .. branch: py_ssize_t Explicitly use Py_ssize_t as the Signed type in pypy c-api + +.. branch: cpyext-jit + +Differentiate the code to call METH_NOARGS, METH_O and METH_VARARGS in cpyext: +this allows to write specialized code which is much faster than previous +completely generic version. Moreover, let the JIT to look inside the cpyext +module: the net result is that cpyext calls are up to 7x faster. However, this +is true only for very simple situations: in all real life code, we are still +much slower than CPython (more optimizations to come) diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -11,7 +11,7 @@ To build pypy-c you need a working python environment, and a C compiler. It is possible to translate with a CPython 2.6 or later, but this is not -the preferred way, because it will take a lot longer to run � depending +the preferred way, because it will take a lot longer to run – depending on your architecture, between two and three times as long. So head to `our downloads`_ and get the latest stable version. @@ -25,8 +25,10 @@ This compiler, while the standard one for Python 2.7, is deprecated. Microsoft has made it available as the `Microsoft Visual C++ Compiler for Python 2.7`_ (the link -was checked in Nov 2016). Note that the compiler suite will be installed in -``C:\Users\\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python``. +was checked in Nov 2016). Note that the compiler suite may be installed in +``C:\Users\\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python`` +or in +``C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python``. A current version of ``setuptools`` will be able to find it there. For Windows 10, you must right-click the download, and under ``Properties`` -> ``Compatibility`` mark it as ``Run run this program in comatibility mode for`` @@ -41,7 +43,6 @@ ----------------------------------- We routinely test translation using v9, also known as Visual Studio 2008. -Our buildbot is still using the Express Edition, not the compiler noted above. Other configurations may work as well. The translation scripts will set up the appropriate environment variables @@ -81,6 +82,31 @@ .. _build instructions: http://pypy.org/download.html#building-from-source +Setting Up Visual Studio for building SSL in Python3 +---------------------------------------------------- + +On Python3, the ``ssl`` module is based on ``cffi``, and requires a build step after +translation. However ``distutils`` does not support the Micorosft-provided Visual C +compiler, and ``cffi`` depends on ``distutils`` to find the compiler. The +traditional solution to this problem is to install the ``setuptools`` module +via running ``-m ensurepip`` which installs ``pip`` and ``setuptools``. However +``pip`` requires ``ssl``. So we have a chicken-and-egg problem: ``ssl`` depends on +``cffi`` which depends on ``setuptools``, which depends on ``ensurepip``, which +depends on ``ssl``. + +In order to solve this, the buildbot sets an environment varaible that helps +``distutils`` find the compiler without ``setuptools``:: + + set VS90COMNTOOLS=C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python\9.0\VC\bin + +or whatever is appropriate for your machine. Note that this is not enough, you +must also copy the ``vcvarsall.bat`` file fron the ``...\9.0`` directory to the +``...\9.0\VC`` directory, and edit it, changing the lines that set +``VCINSTALLDIR`` and ``WindowsSdkDir``:: + + set VCINSTALLDIR=%~dp0\ + set WindowsSdkDir=%~dp0\..\WinSDK\ + Preparing Windows for the large build ------------------------------------- diff --git a/pypy/goal/getnightly.py b/pypy/goal/getnightly.py --- a/pypy/goal/getnightly.py +++ b/pypy/goal/getnightly.py @@ -15,7 +15,7 @@ arch = 'linux' cmd = 'wget "%s"' TAR_OPTIONS += ' --wildcards' - binfiles = "'*/bin/pypy' '*/bin/libpypy-c.so'" + binfiles = "'*/bin/pypy*' '*/bin/libpypy-c.so*'" if os.uname()[-1].startswith('arm'): arch += '-armhf-raspbian' elif sys.platform.startswith('darwin'): diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py --- a/pypy/interpreter/astcompiler/test/test_astbuilder.py +++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py @@ -1246,3 +1246,7 @@ exc = py.test.raises(SyntaxError, self.get_ast, input).value assert exc.msg == ("(unicode error) 'unicodeescape' codec can't decode" " bytes in position 0-1: truncated \\xXX escape") + input = "u'\\x1'" + exc = py.test.raises(SyntaxError, self.get_ast, input).value + assert exc.msg == ("(unicode error) 'unicodeescape' codec can't decode" + " bytes in position 0-2: truncated \\xXX escape") diff --git a/pypy/interpreter/pyparser/future.py b/pypy/interpreter/pyparser/future.py --- a/pypy/interpreter/pyparser/future.py +++ b/pypy/interpreter/pyparser/future.py @@ -85,13 +85,17 @@ # permissive parsing of the given list of tokens; it relies on # the real parsing done afterwards to give errors. it.skip_newlines() - it.skip_name("r") or it.skip_name("u") or it.skip_name("ru") - if it.skip(pygram.tokens.STRING): - it.skip_newlines() - while (it.skip_name("from") and + docstring_possible = True + while True: + it.skip_name("r") or it.skip_name("u") or it.skip_name("ru") + if docstring_possible and it.skip(pygram.tokens.STRING): + it.skip_newlines() + docstring_possible = False + if not (it.skip_name("from") and it.skip_name("__future__") and it.skip_name("import")): + break it.skip(pygram.tokens.LPAR) # optionally # return in 'last_position' any line-column pair that points # somewhere inside the last __future__ import statement diff --git a/pypy/interpreter/pyparser/test/test_future.py b/pypy/interpreter/pyparser/test/test_future.py --- a/pypy/interpreter/pyparser/test/test_future.py +++ b/pypy/interpreter/pyparser/test/test_future.py @@ -208,3 +208,13 @@ 'from __future__ import with_statement;') f = run(s, (2, 23)) assert f == fut.CO_FUTURE_DIVISION | fut.CO_FUTURE_WITH_STATEMENT + +def test_future_doc_future(): + # for some reason people do this :-[ + s = ''' +from __future__ import generators +"Docstring" +from __future__ import division + ''' + f = run(s, (4, 24)) + assert f == fut.CO_FUTURE_DIVISION | fut.CO_GENERATOR_ALLOWED diff --git a/pypy/interpreter/test/test_unicodehelper.py b/pypy/interpreter/test/test_unicodehelper.py --- a/pypy/interpreter/test/test_unicodehelper.py +++ b/pypy/interpreter/test/test_unicodehelper.py @@ -1,4 +1,7 @@ -from pypy.interpreter.unicodehelper import encode_utf8, decode_utf8 +import pytest +import struct +from pypy.interpreter.unicodehelper import ( + encode_utf8, decode_utf8, unicode_encode_utf_32_be) class FakeSpace: pass @@ -24,3 +27,23 @@ assert map(ord, got) == [0xd800, 0xdc00] got = decode_utf8(space, "\xf0\x90\x80\x80") assert map(ord, got) == [0x10000] + + at pytest.mark.parametrize('unich', [u"\ud800", u"\udc80"]) +def test_utf32_surrogates(unich): + assert (unicode_encode_utf_32_be(unich, 1, None) == + struct.pack('>i', ord(unich))) + with pytest.raises(UnicodeEncodeError): + unicode_encode_utf_32_be(unich, 1, None, allow_surrogates=False) + + def replace_with(ru, rs): + def errorhandler(errors, enc, msg, u, startingpos, endingpos): + if errors == 'strict': + raise UnicodeEncodeError(enc, u, startingpos, endingpos, msg) + return ru, rs, endingpos + return unicode_encode_utf_32_be( + u"<%s>" % unich, 3, None, + errorhandler, allow_surrogates=False) + + assert replace_with(u'rep', None) == u''.encode('utf-32-be') + assert (replace_with(None, '\xca\xfe\xca\xfe') == + '\x00\x00\x00<\xca\xfe\xca\xfe\x00\x00\x00>') diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -1,7 +1,11 @@ +from rpython.rlib.objectmodel import specialize +from rpython.rlib.rarithmetic import intmask +from rpython.rlib.rstring import StringBuilder, UnicodeBuilder +from rpython.rlib import runicode +from rpython.rlib.runicode import ( + default_unicode_error_encode, default_unicode_error_decode, + MAXUNICODE, BYTEORDER, BYTEORDER2, UNICHR) from pypy.interpreter.error import OperationError -from rpython.rlib.objectmodel import specialize -from rpython.rlib import runicode -from pypy.module._codecs import interp_codecs @specialize.memo() def decode_error_handler(space): @@ -37,6 +41,7 @@ # These functions take and return unwrapped rpython strings and unicodes def decode_unicode_escape(space, string): + from pypy.module._codecs import interp_codecs state = space.fromcache(interp_codecs.CodecState) unicodedata_handler = state.get_unicodedata_handler(space) result, consumed = runicode.str_decode_unicode_escape( @@ -71,3 +76,229 @@ uni, len(uni), "strict", errorhandler=None, allow_surrogates=True) + +# ____________________________________________________________ +# utf-32 + +def str_decode_utf_32(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "native") + return result, length + +def str_decode_utf_32_be(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "big") + return result, length + +def str_decode_utf_32_le(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "little") + return result, length + +def py3k_str_decode_utf_32(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "native", 'utf-32-' + BYTEORDER2) + return result, length + +def py3k_str_decode_utf_32_be(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "big", 'utf-32-be') + return result, length + +def py3k_str_decode_utf_32_le(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "little", 'utf-32-le') + return result, length + +BOM32_DIRECT = intmask(0x0000FEFF) +BOM32_REVERSE = intmask(0xFFFE0000) + +def str_decode_utf_32_helper(s, size, errors, final=True, + errorhandler=None, + byteorder="native", + public_encoding_name='utf32'): + if errorhandler is None: + errorhandler = default_unicode_error_decode + bo = 0 + + if BYTEORDER == 'little': + iorder = [0, 1, 2, 3] + else: + iorder = [3, 2, 1, 0] + + # Check for BOM marks (U+FEFF) in the input and adjust current + # byte order setting accordingly. In native mode, the leading BOM + # mark is skipped, in all other modes, it is copied to the output + # stream as-is (giving a ZWNBSP character). + pos = 0 + if byteorder == 'native': + if size >= 4: + bom = intmask( + (ord(s[iorder[3]]) << 24) | (ord(s[iorder[2]]) << 16) | + (ord(s[iorder[1]]) << 8) | ord(s[iorder[0]])) + if BYTEORDER == 'little': + if bom == BOM32_DIRECT: + pos += 4 + bo = -1 + elif bom == BOM32_REVERSE: + pos += 4 + bo = 1 + else: + if bom == BOM32_DIRECT: + pos += 4 + bo = 1 + elif bom == BOM32_REVERSE: + pos += 4 + bo = -1 + elif byteorder == 'little': + bo = -1 + else: + bo = 1 + if size == 0: + return u'', 0, bo + if bo == -1: + # force little endian + iorder = [0, 1, 2, 3] + elif bo == 1: + # force big endian + iorder = [3, 2, 1, 0] + + result = UnicodeBuilder(size // 4) + + while pos < size: + # remaining bytes at the end? (size should be divisible by 4) + if len(s) - pos < 4: + if not final: + break + r, pos = errorhandler(errors, public_encoding_name, + "truncated data", + s, pos, len(s)) + result.append(r) + if len(s) - pos < 4: + break + continue + ch = ((ord(s[pos + iorder[3]]) << 24) | (ord(s[pos + iorder[2]]) << 16) | + (ord(s[pos + iorder[1]]) << 8) | ord(s[pos + iorder[0]])) + if ch >= 0x110000: + r, pos = errorhandler(errors, public_encoding_name, + "codepoint not in range(0x110000)", + s, pos, len(s)) + result.append(r) + continue + + if MAXUNICODE < 65536 and ch >= 0x10000: + ch -= 0x10000L + result.append(unichr(0xD800 + (ch >> 10))) + result.append(unichr(0xDC00 + (ch & 0x03FF))) + else: + result.append(UNICHR(ch)) + pos += 4 + return result.build(), pos, bo + +def _STORECHAR32(result, CH, byteorder): + c0 = chr(((CH) >> 24) & 0xff) + c1 = chr(((CH) >> 16) & 0xff) + c2 = chr(((CH) >> 8) & 0xff) + c3 = chr((CH) & 0xff) + if byteorder == 'little': + result.append(c3) + result.append(c2) + result.append(c1) + result.append(c0) + else: + result.append(c0) + result.append(c1) + result.append(c2) + result.append(c3) + +def unicode_encode_utf_32_helper(s, size, errors, + errorhandler=None, + allow_surrogates=True, + byteorder='little', + public_encoding_name='utf32'): + if errorhandler is None: + errorhandler = default_unicode_error_encode + if size == 0: + if byteorder == 'native': + result = StringBuilder(4) + _STORECHAR32(result, 0xFEFF, BYTEORDER) + return result.build() + return "" + + result = StringBuilder(size * 4 + 4) + if byteorder == 'native': + _STORECHAR32(result, 0xFEFF, BYTEORDER) + byteorder = BYTEORDER + + pos = 0 + while pos < size: + ch = ord(s[pos]) + pos += 1 + ch2 = 0 + if not allow_surrogates and 0xD800 <= ch < 0xE000: + ru, rs, pos = errorhandler( + errors, public_encoding_name, 'surrogates not allowed', + s, pos - 1, pos) + if rs is not None: + # py3k only + if len(rs) % 4 != 0: + errorhandler( + 'strict', public_encoding_name, 'surrogates not allowed', + s, pos - 1, pos) + result.append(rs) + continue + for ch in ru: + if ord(ch) < 0xD800: + _STORECHAR32(result, ord(ch), byteorder) + else: + errorhandler( + 'strict', public_encoding_name, + 'surrogates not allowed', s, pos - 1, pos) + continue + if 0xD800 <= ch < 0xDC00 and MAXUNICODE < 65536 and pos < size: + ch2 = ord(s[pos]) + if 0xDC00 <= ch2 < 0xE000: + ch = (((ch & 0x3FF) << 10) | (ch2 & 0x3FF)) + 0x10000 + pos += 1 + _STORECHAR32(result, ch, byteorder) + + return result.build() + +def unicode_encode_utf_32(s, size, errors, + errorhandler=None, allow_surrogates=True): + return unicode_encode_utf_32_helper(s, size, errors, errorhandler, + allow_surrogates, "native") + +def unicode_encode_utf_32_be(s, size, errors, + errorhandler=None, allow_surrogates=True): + return unicode_encode_utf_32_helper(s, size, errors, errorhandler, + allow_surrogates, "big") + +def unicode_encode_utf_32_le(s, size, errors, + errorhandler=None, allow_surrogates=True): + return unicode_encode_utf_32_helper(s, size, errors, errorhandler, + allow_surrogates, "little") + +def py3k_unicode_encode_utf_32(s, size, errors, + errorhandler=None, allow_surrogates=True): + return unicode_encode_utf_32_helper(s, size, errors, errorhandler, + allow_surrogates, "native", + 'utf-32-' + BYTEORDER2) + +def py3k_unicode_encode_utf_32_be(s, size, errors, + errorhandler=None, allow_surrogates=True): + return unicode_encode_utf_32_helper(s, size, errors, errorhandler, + allow_surrogates, "big", + 'utf-32-be') + +def py3k_unicode_encode_utf_32_le(s, size, errors, + errorhandler=None, allow_surrogates=True): + return unicode_encode_utf_32_helper(s, size, errors, errorhandler, + allow_surrogates, "little", + 'utf-32-le') diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py --- a/pypy/module/__builtin__/test/test_builtin.py +++ b/pypy/module/__builtin__/test/test_builtin.py @@ -404,6 +404,7 @@ def test_cmp(self): + assert cmp(float('nan'), float('nan')) == 0 assert cmp(9,9) == 0 assert cmp(0,9) < 0 assert cmp(9,0) > 0 diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi from rpython.rtyper.lltypesystem import rffi -VERSION = "1.11.1" +VERSION = "1.11.3" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.11.1", ("This test_c.py file is for testing a version" +assert __version__ == "1.11.3", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py --- a/pypy/module/_cffi_backend/test/test_recompiler.py +++ b/pypy/module/_cffi_backend/test/test_recompiler.py @@ -8,7 +8,8 @@ @unwrap_spec(cdef='text', module_name='text', source='text', packed=int) def prepare(space, cdef, module_name, source, w_includes=None, - w_extra_source=None, w_min_version=None, packed=False): + w_extra_source=None, w_min_version=None, packed=False, From pypy.commits at gmail.com Fri Jan 12 12:34:00 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 12 Jan 2018 09:34:00 -0800 (PST) Subject: [pypy-commit] pypy py3.5-refactor-slots: hg merge refactor-slots Message-ID: <5a58f188.14121c0a.fcf65.c32e@mx.google.com> Author: Ronan Lamy Branch: py3.5-refactor-slots Changeset: r93658:5eda67de1f64 Date: 2017-10-02 17:48 +0200 http://bitbucket.org/pypy/pypy/changeset/5eda67de1f64/ Log: hg merge refactor-slots diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -418,39 +418,19 @@ return space.newint(generic_cpy_call(space, func_target, w_self, w_other)) -from rpython.rlib.nonconst import NonConstant +SLOT_FACTORIES = {} +def slot_factory(tp_name): + def decorate(func): + SLOT_FACTORIES[tp_name] = func + return func + return decorate -def build_slot_tp_function(space, typedef, name): + +def build_slot_tp_function(space, typedef, name, method_name): w_type = space.gettypeobject(typedef) handled = False # unary functions - for tp_name, attr in [('tp_as_async.c_am_await', '__await__'), - ('tp_as_async.c_am_anext', '__anext__'), - ('tp_as_async.c_am_aiter', '__aiter__'), - ('tp_as_number.c_nb_int', '__int__'), - ('tp_as_number.c_nb_long', '__long__'), - ('tp_as_number.c_nb_float', '__float__'), - ('tp_as_number.c_nb_negative', '__neg__'), - ('tp_as_number.c_nb_positive', '__pos__'), - ('tp_as_number.c_nb_absolute', '__abs__'), - ('tp_as_number.c_nb_invert', '__invert__'), - ('tp_as_number.c_nb_index', '__index__'), - ('tp_str', '__str__'), - ('tp_repr', '__repr__'), - ('tp_iter', '__iter__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - - @slot_function([PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self): - return space.call_function(slot_fn, w_self) - handled = True - for tp_name, attr in [('tp_hash', '__hash__'), ('tp_as_sequence.c_sq_length', '__len__'), ('tp_as_mapping.c_mp_length', '__len__'), @@ -669,27 +649,8 @@ w_obj = space.w_None return space.call_function(get_fn, w_self, w_obj, w_value) slot_func = slot_tp_descr_get - elif name == 'tp_descr_set': - set_fn = w_type.lookup('__set__') - delete_fn = w_type.lookup('__delete__') - if set_fn is None and delete_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_descr_set(space, w_self, w_obj, w_value): - if w_value is not None: - if set_fn is None: - raise oefmt(space.w_TypeError, - "%s object has no __set__", typedef.name) - space.call_function(set_fn, w_self, w_obj, w_value) - else: - if delete_fn is None: - raise oefmt(space.w_TypeError, - "%s object has no __delete__", typedef.name) - space.call_function(delete_fn, w_self, w_obj) - return 0 - slot_func = slot_tp_descr_set + elif name in SLOT_FACTORIES: + return SLOT_FACTORIES[name](space, typedef, name, method_name) else: # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce # tp_as_sequence.c_sq_contains, tp_as_sequence.c_sq_length @@ -698,6 +659,64 @@ return slot_func +def make_unary_slot(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_fn = w_type.lookup(attr) + if slot_fn is None: + return + + @slot_function([PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self): + return space.call_function(slot_fn, w_self) + return slot_func + + +UNARY_SLOTS = [ + 'tp_as_async.c_am_await', + 'tp_as_async.c_am_anext', + 'tp_as_async.c_am_aiter', + 'tp_as_number.c_nb_int', + 'tp_as_number.c_nb_long', + 'tp_as_number.c_nb_float', + 'tp_as_number.c_nb_negative', + 'tp_as_number.c_nb_positive', + 'tp_as_number.c_nb_absolute', + 'tp_as_number.c_nb_invert', + 'tp_as_number.c_nb_index', + 'tp_as_number.c_nb_hex', + 'tp_as_number.c_nb_oct', + 'tp_str', + 'tp_repr', + 'tp_iter'] +for name in UNARY_SLOTS: + slot_factory(name)(make_unary_slot) + + at slot_factory('tp_descr_set') +def make_tp_descr_set(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + name = 'descr_set' + set_fn = w_type.lookup('__set__') + delete_fn = w_type.lookup('__delete__') + if set_fn is None and delete_fn is None: + return + + @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_tp_descr_set(space, w_self, w_obj, w_value): + if w_value is not None: + if set_fn is None: + raise oefmt(space.w_TypeError, + "%s object has no __set__", typedef.name) + space.call_function(set_fn, w_self, w_obj, w_value) + else: + if delete_fn is None: + raise oefmt(space.w_TypeError, + "%s object has no __delete__", typedef.name) + space.call_function(delete_fn, w_self, w_obj) + return 0 + return slot_tp_descr_set + def slot_from___buffer__(space, typedef, buff_fn): name = 'bf_getbuffer' diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -228,7 +228,7 @@ SLOTS = {} @specialize.memo() -def get_slot_tp_function(space, typedef, name): +def get_slot_tp_function(space, typedef, name, method_name): """Return a description of the slot C function to use for the built-in type for 'typedef'. The 'name' is the slot name. This is a memo function that, after translation, returns one of a built-in finite set. @@ -237,7 +237,7 @@ try: return SLOTS[key] except KeyError: - slot_func = build_slot_tp_function(space, typedef, name) + slot_func = build_slot_tp_function(space, typedef, name, method_name) api_func = slot_func.api_func if slot_func else None SLOTS[key] = api_func return api_func @@ -281,7 +281,7 @@ def update_all_slots_builtin(space, w_type, pto): typedef = w_type.layout.typedef for method_name, slot_name, slot_names, slot_apifunc in slotdefs_for_tp_slots: - slot_apifunc = get_slot_tp_function(space, typedef, slot_name) + slot_apifunc = get_slot_tp_function(space, typedef, slot_name, method_name) if not slot_apifunc: warn_missing_slot(space, method_name, slot_name, w_type) continue From pypy.commits at gmail.com Fri Jan 12 12:35:55 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 12 Jan 2018 09:35:55 -0800 (PST) Subject: [pypy-commit] pypy py3.5-refactor-slots: hg merge py3.5 Message-ID: <5a58f1fb.07c5df0a.53032.3d82@mx.google.com> Author: Ronan Lamy Branch: py3.5-refactor-slots Changeset: r93659:be7fb656f9f4 Date: 2018-01-12 17:35 +0000 http://bitbucket.org/pypy/pypy/changeset/be7fb656f9f4/ Log: hg merge py3.5 diff too long, truncating to 2000 out of 110564 lines diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -59,6 +59,7 @@ ^rpython/rlib/rvmprof/src/shared/libbacktrace/config.h$ ^rpython/rlib/rvmprof/src/shared/libbacktrace/config.log$ ^rpython/rlib/rvmprof/src/shared/libbacktrace/config.status$ +^pypy/tool/dest$ ^pypy/goal/pypy-translation-snapshot$ ^pypy/goal/pypy-c ^pypy/goal/pypy3-c @@ -75,6 +76,8 @@ ^lib_pypy/.+.c$ ^lib_pypy/.+.o$ ^lib_pypy/.+.so$ +^lib_pypy/.+.pyd$ +^lib_pypy/Release/ ^pypy/doc/discussion/.+\.html$ ^include/.+\.h$ ^include/.+\.inl$ diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -40,3 +40,13 @@ 2875f328eae2216a87f3d6f335092832eb031f56 release-pypy3.5-v5.7.1 c925e73810367cd960a32592dd7f728f436c125c release-pypy2.7-v5.8.0 a37ecfe5f142bc971a86d17305cc5d1d70abec64 release-pypy3.5-v5.8.0 +03d614975835870da65ff0481e1edad68ebbcb8d release-pypy2.7-v5.9.0 +d72f9800a42b46a8056951b1da2426d2c2d8d502 release-pypy3.5-v5.9.0 +03d614975835870da65ff0481e1edad68ebbcb8d release-pypy2.7-v5.9.0 +84a2f3e6a7f88f2fe698e473998755b3bd1a12e2 release-pypy2.7-v5.9.0 +0e7ea4fe15e82d5124e805e2e4a37cae1a402d4b release-pypy2.7-v5.10.0 +a91df6163fb76df245091f741dbf6a23ddc72374 release-pypy3.5-v5.10.0 +a91df6163fb76df245091f741dbf6a23ddc72374 release-pypy3.5-v5.10.0 +0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0 +0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0 +09f9160b643e3f02ccb8c843b2fbb4e5cbf54082 release-pypy3.5-v5.10.0 diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -30,7 +30,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2017 +PyPy Copyright holders 2003-2018 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at @@ -339,8 +339,10 @@ Stanisław Halik Julien Phalip Roman Podoliaka + Steve Papanik Eli Stevens Boglarka Vezer + gabrielg PavloKapyshin Tomer Chachamu Christopher Groskopf @@ -363,11 +365,13 @@ Konrad Delong Dinu Gherman pizi + Tomáš Pružina James Robert Armin Ronacher Diana Popa Mads Kiilerich Brett Cannon + Caleb Hattingh aliceinwire Zooko Wilcox-O Hearn James Lan @@ -388,6 +392,7 @@ Jason Madden Yaroslav Fedevych Even Wiik Thomassen + m at funkyhat.org Stefan Marr Heinrich-Heine University, Germany diff --git a/_pytest/terminal.py b/_pytest/terminal.py --- a/_pytest/terminal.py +++ b/_pytest/terminal.py @@ -366,11 +366,11 @@ EXIT_OK, EXIT_TESTSFAILED, EXIT_INTERRUPTED, EXIT_USAGEERROR, EXIT_NOTESTSCOLLECTED) if exitstatus in summary_exit_codes: - self.config.hook.pytest_terminal_summary(terminalreporter=self) self.summary_errors() self.summary_failures() self.summary_warnings() self.summary_passes() + self.config.hook.pytest_terminal_summary(terminalreporter=self) if exitstatus == EXIT_INTERRUPTED: self._report_keyboardinterrupt() del self._keyboardinterrupt_memo diff --git a/extra_tests/requirements.txt b/extra_tests/requirements.txt new file mode 100644 --- /dev/null +++ b/extra_tests/requirements.txt @@ -0,0 +1,3 @@ +pytest +hypothesis +vmprof diff --git a/extra_tests/test_bytes.py b/extra_tests/test_bytes.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_bytes.py @@ -0,0 +1,84 @@ +from hypothesis import strategies as st +from hypothesis import given, example + +st_bytestring = st.binary() | st.binary().map(bytearray) + + at given(st_bytestring, st_bytestring, st_bytestring) +def test_find(u, prefix, suffix): + s = prefix + u + suffix + assert 0 <= s.find(u) <= len(prefix) + assert s.find(u, len(prefix), len(s) - len(suffix)) == len(prefix) + + at given(st_bytestring, st_bytestring, st_bytestring) +def test_index(u, prefix, suffix): + s = prefix + u + suffix + assert 0 <= s.index(u) <= len(prefix) + assert s.index(u, len(prefix), len(s) - len(suffix)) == len(prefix) + + at given(st_bytestring, st_bytestring, st_bytestring) +def test_rfind(u, prefix, suffix): + s = prefix + u + suffix + assert s.rfind(u) >= len(prefix) + assert s.rfind(u, len(prefix), len(s) - len(suffix)) == len(prefix) + + at given(st_bytestring, st_bytestring, st_bytestring) +def test_rindex(u, prefix, suffix): + s = prefix + u + suffix + assert s.rindex(u) >= len(prefix) + assert s.rindex(u, len(prefix), len(s) - len(suffix)) == len(prefix) + +def adjust_indices(u, start, end): + if end < 0: + end = max(end + len(u), 0) + else: + end = min(end, len(u)) + if start < 0: + start = max(start + len(u), 0) + return start, end + + at given(st_bytestring, st_bytestring) +def test_startswith_basic(u, v): + assert u.startswith(v) is (u[:len(v)] == v) + + at example(b'x', b'', 1) + at example(b'x', b'', 2) + at given(st_bytestring, st_bytestring, st.integers()) +def test_startswith_start(u, v, start): + expected = u[start:].startswith(v) if v else (start <= len(u)) + assert u.startswith(v, start) is expected + + at example(b'x', b'', 1, 0) + at example(b'xx', b'', -1, 0) + at given(st_bytestring, st_bytestring, st.integers(), st.integers()) +def test_startswith_3(u, v, start, end): + if v: + expected = u[start:end].startswith(v) + else: # CPython leaks implementation details in this case + start0, end0 = adjust_indices(u, start, end) + expected = start0 <= len(u) and start0 <= end0 + assert u.startswith(v, start, end) is expected + + at given(st_bytestring, st_bytestring) +def test_endswith_basic(u, v): + if len(v) > len(u): + assert u.endswith(v) is False + else: + assert u.endswith(v) is (u[len(u) - len(v):] == v) + + at example(b'x', b'', 1) + at example(b'x', b'', 2) + at given(st_bytestring, st_bytestring, st.integers()) +def test_endswith_2(u, v, start): + expected = u[start:].endswith(v) if v else (start <= len(u)) + assert u.endswith(v, start) is expected + + at example(b'x', b'', 1, 0) + at example(b'xx', b'', -1, 0) + at given(st_bytestring, st_bytestring, st.integers(), st.integers()) +def test_endswith_3(u, v, start, end): + if v: + expected = u[start:end].endswith(v) + else: # CPython leaks implementation details in this case + start0, end0 = adjust_indices(u, start, end) + expected = start0 <= len(u) and start0 <= end0 + assert u.endswith(v, start, end) is expected diff --git a/extra_tests/test_import.py b/extra_tests/test_import.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_import.py @@ -0,0 +1,41 @@ +import pytest +import sys +import time +from _thread import start_new_thread + + at pytest.mark.xfail('__pypy__' not in sys.builtin_module_names, + reason='Fails on CPython') +def test_multithreaded_import(tmpdir): + tmpfile = tmpdir.join('multithreaded_import_test.py') + tmpfile.write('''if 1: + x = 666 + import time + for i in range(1000): time.sleep(0.001) + x = 42 + ''') + + oldpath = sys.path[:] + try: + sys.path.insert(0, str(tmpdir)) + got = [] + + def check(): + import multithreaded_import_test + got.append(getattr(multithreaded_import_test, 'x', '?')) + + for i in range(5): + start_new_thread(check, ()) + + for n in range(100): + for i in range(105): + time.sleep(0.001) + if len(got) == 5: + break + else: + raise AssertionError("got %r so far but still waiting" % + (got,)) + + assert got == [42] * 5 + + finally: + sys.path[:] = oldpath diff --git a/extra_tests/test_json.py b/extra_tests/test_json.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_json.py @@ -0,0 +1,29 @@ +import pytest +import json +from hypothesis import given, strategies + +def is_(x, y): + return type(x) is type(y) and x == y + +def test_no_ensure_ascii(): + assert is_(json.dumps(u"\u1234", ensure_ascii=False), u'"\u1234"') + assert is_(json.dumps(u"\xc0", ensure_ascii=False), u'"\xc0"') + with pytest.raises(TypeError): + json.dumps((u"\u1234", b"x"), ensure_ascii=False) + with pytest.raises(TypeError): + json.dumps((b"x", u"\u1234"), ensure_ascii=False) + +def test_issue2191(): + assert is_(json.dumps(u"xxx", ensure_ascii=False), u'"xxx"') + +jsondata = strategies.recursive( + strategies.none() | + strategies.booleans() | + strategies.floats(allow_nan=False) | + strategies.text(), + lambda children: strategies.lists(children) | + strategies.dictionaries(strategies.text(), children)) + + at given(jsondata) +def test_roundtrip(d): + assert json.loads(json.dumps(d)) == d diff --git a/extra_tests/test_textio.py b/extra_tests/test_textio.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_textio.py @@ -0,0 +1,48 @@ +from hypothesis import given, strategies as st + +from io import BytesIO, TextIOWrapper +import os + +def translate_newlines(text): + text = text.replace('\r\n', '\n') + text = text.replace('\r', '\n') + return text.replace('\n', os.linesep) + + at st.composite +def st_readline_universal( + draw, st_nlines=st.integers(min_value=0, max_value=10)): + n_lines = draw(st_nlines) + lines = draw(st.lists( + st.text(st.characters(blacklist_characters='\r\n')), + min_size=n_lines, max_size=n_lines)) + limits = [] + for line in lines: + limit = draw(st.integers(min_value=0, max_value=len(line) + 5)) + limits.append(limit) + limits.append(-1) + endings = draw(st.lists( + st.sampled_from(['\n', '\r', '\r\n']), + min_size=n_lines, max_size=n_lines)) + return ( + ''.join(line + ending for line, ending in zip(lines, endings)), + limits) + + at given(data=st_readline_universal(), + mode=st.sampled_from(['\r', '\n', '\r\n', '', None])) +def test_readline(data, mode): + txt, limits = data + textio = TextIOWrapper( + BytesIO(txt.encode('utf-8', 'surrogatepass')), + encoding='utf-8', errors='surrogatepass', newline=mode) + lines = [] + for limit in limits: + line = textio.readline(limit) + if limit >= 0: + assert len(line) <= limit + if line: + lines.append(line) + elif limit: + break + if mode is None: + txt = translate_newlines(txt) + assert txt.startswith(u''.join(lines)) diff --git a/extra_tests/test_unicode.py b/extra_tests/test_unicode.py --- a/extra_tests/test_unicode.py +++ b/extra_tests/test_unicode.py @@ -1,3 +1,4 @@ +import sys import pytest from hypothesis import strategies as st from hypothesis import given, settings, example @@ -32,3 +33,89 @@ @given(s=st.text()) def test_composition(s, norm1, norm2, norm3): assert normalize(norm2, normalize(norm1, s)) == normalize(norm3, s) + + at given(st.text(), st.text(), st.text()) +def test_find(u, prefix, suffix): + s = prefix + u + suffix + assert 0 <= s.find(u) <= len(prefix) + assert s.find(u, len(prefix), len(s) - len(suffix)) == len(prefix) + + at given(st.text(), st.text(), st.text()) +def test_index(u, prefix, suffix): + s = prefix + u + suffix + assert 0 <= s.index(u) <= len(prefix) + assert s.index(u, len(prefix), len(s) - len(suffix)) == len(prefix) + + at given(st.text(), st.text(), st.text()) +def test_rfind(u, prefix, suffix): + s = prefix + u + suffix + assert s.rfind(u) >= len(prefix) + assert s.rfind(u, len(prefix), len(s) - len(suffix)) == len(prefix) + + at given(st.text(), st.text(), st.text()) +def test_rindex(u, prefix, suffix): + s = prefix + u + suffix + assert s.rindex(u) >= len(prefix) + assert s.rindex(u, len(prefix), len(s) - len(suffix)) == len(prefix) + +def adjust_indices(u, start, end): + if end < 0: + end = max(end + len(u), 0) + else: + end = min(end, len(u)) + if start < 0: + start = max(start + len(u), 0) + return start, end + + at given(st.text(), st.text()) +def test_startswith_basic(u, v): + assert u.startswith(v) is (u[:len(v)] == v) + + at example(u'x', u'', 1) + at example(u'x', u'', 2) + at given(st.text(), st.text(), st.integers()) +def test_startswith_2(u, v, start): + if v or sys.version_info[0] == 2: + expected = u[start:].startswith(v) + else: # CPython leaks implementation details in this case + expected = start <= len(u) + assert u.startswith(v, start) is expected + + at example(u'x', u'', 1, 0) + at example(u'xx', u'', -1, 0) + at given(st.text(), st.text(), st.integers(), st.integers()) +def test_startswith_3(u, v, start, end): + if v or sys.version_info[0] == 2: + expected = u[start:end].startswith(v) + else: # CPython leaks implementation details in this case + start0, end0 = adjust_indices(u, start, end) + expected = start0 <= len(u) and start0 <= end0 + assert u.startswith(v, start, end) is expected + + at given(st.text(), st.text()) +def test_endswith_basic(u, v): + if len(v) > len(u): + assert u.endswith(v) is False + else: + assert u.endswith(v) is (u[len(u) - len(v):] == v) + + at example(u'x', u'', 1) + at example(u'x', u'', 2) + at given(st.text(), st.text(), st.integers()) +def test_endswith_2(u, v, start): + if v or sys.version_info[0] == 2: + expected = u[start:].endswith(v) + else: # CPython leaks implementation details in this case + expected = start <= len(u) + assert u.endswith(v, start) is expected + + at example(u'x', u'', 1, 0) + at example(u'xx', u'', -1, 0) + at given(st.text(), st.text(), st.integers(), st.integers()) +def test_endswith_3(u, v, start, end): + if v or sys.version_info[0] == 2: + expected = u[start:end].endswith(v) + else: # CPython leaks implementation details in this case + start0, end0 = adjust_indices(u, start, end) + expected = start0 <= len(u) and start0 <= end0 + assert u.endswith(v, start, end) is expected diff --git a/extra_tests/test_vmprof_greenlet.py b/extra_tests/test_vmprof_greenlet.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_vmprof_greenlet.py @@ -0,0 +1,28 @@ +import time +import pytest +import greenlet +vmprof = pytest.importorskip('vmprof') + +def count_samples(filename): + stats = vmprof.read_profile(filename) + return len(stats.profiles) + +def cpuburn(duration): + end = time.time() + duration + while time.time() < end: + pass + +def test_sampling_inside_callback(tmpdir): + # see also test_sampling_inside_callback inside + # pypy/module/_continuation/test/test_stacklet.py + # + G = greenlet.greenlet(cpuburn) + fname = tmpdir.join('log.vmprof') + with fname.open('w+b') as f: + vmprof.enable(f.fileno(), 1/250.0) + G.switch(0.1) + vmprof.disable() + + samples = count_samples(str(fname)) + # 0.1 seconds at 250Hz should be 25 samples + assert 23 < samples < 27 diff --git a/lib-python/2.7/ctypes/__init__.py b/lib-python/2.7/ctypes/__init__.py --- a/lib-python/2.7/ctypes/__init__.py +++ b/lib-python/2.7/ctypes/__init__.py @@ -360,14 +360,15 @@ self._FuncPtr = _FuncPtr if handle is None: - if flags & _FUNCFLAG_CDECL: - pypy_dll = _ffi.CDLL(name, mode) - else: - pypy_dll = _ffi.WinDLL(name, mode) - self.__pypy_dll__ = pypy_dll - handle = int(pypy_dll) - if _sys.maxint > 2 ** 32: - handle = int(handle) # long -> int + handle = 0 + if flags & _FUNCFLAG_CDECL: + pypy_dll = _ffi.CDLL(name, mode, handle) + else: + pypy_dll = _ffi.WinDLL(name, mode, handle) + self.__pypy_dll__ = pypy_dll + handle = int(pypy_dll) + if _sys.maxint > 2 ** 32: + handle = int(handle) # long -> int self._handle = handle def __repr__(self): diff --git a/lib-python/2.7/inspect.py b/lib-python/2.7/inspect.py --- a/lib-python/2.7/inspect.py +++ b/lib-python/2.7/inspect.py @@ -40,6 +40,10 @@ import linecache from operator import attrgetter from collections import namedtuple +try: + from cpyext import is_cpyext_function as _is_cpyext_function +except ImportError: + _is_cpyext_function = lambda obj: False # These constants are from Include/code.h. CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS = 0x1, 0x2, 0x4, 0x8 @@ -230,7 +234,7 @@ __doc__ documentation string __name__ original name of this function or method __self__ instance to which a method is bound, or None""" - return isinstance(object, types.BuiltinFunctionType) + return isinstance(object, types.BuiltinFunctionType) or _is_cpyext_function(object) def isroutine(object): """Return true if the object is any kind of function or method.""" diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py --- a/lib-python/2.7/subprocess.py +++ b/lib-python/2.7/subprocess.py @@ -1296,7 +1296,7 @@ 'copyfile' in caller.f_globals): dest_dir = sys.pypy_resolvedirof(target_executable) src_dir = sys.pypy_resolvedirof(sys.executable) - for libname in ['libpypy-c.so', 'libpypy-c.dylib']: + for libname in ['libpypy-c.so', 'libpypy-c.dylib', 'libpypy-c.dll']: dest_library = os.path.join(dest_dir, libname) src_library = os.path.join(src_dir, libname) if os.path.exists(src_library): diff --git a/lib-python/2.7/test/test_urllib2net.py b/lib-python/2.7/test/test_urllib2net.py --- a/lib-python/2.7/test/test_urllib2net.py +++ b/lib-python/2.7/test/test_urllib2net.py @@ -286,7 +286,7 @@ self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 120) u.close() - FTP_HOST = 'ftp://ftp.debian.org/debian/' + FTP_HOST = 'ftp://www.pythontest.net/' def test_ftp_basic(self): self.assertIsNone(socket.getdefaulttimeout()) diff --git a/lib-python/2.7/warnings.py b/lib-python/2.7/warnings.py --- a/lib-python/2.7/warnings.py +++ b/lib-python/2.7/warnings.py @@ -43,11 +43,12 @@ unicodetype = unicode except NameError: unicodetype = () + template = "%s: %s: %s\n" try: message = str(message) except UnicodeEncodeError: - pass - s = "%s: %s: %s\n" % (lineno, category.__name__, message) + template = unicode(template) + s = template % (lineno, category.__name__, message) line = linecache.getline(filename, lineno) if line is None else line if line: line = line.strip() diff --git a/lib-python/3/ctypes/test/test_bitfields.py b/lib-python/3/ctypes/test/test_bitfields.py --- a/lib-python/3/ctypes/test/test_bitfields.py +++ b/lib-python/3/ctypes/test/test_bitfields.py @@ -1,5 +1,5 @@ from ctypes import * -from ctypes.test import need_symbol +from ctypes.test import need_symbol, xfail import unittest import os @@ -279,6 +279,7 @@ self.assertEqual(b, b'\xef\xcd\xab\x21') @need_symbol('c_uint32') + @xfail def test_uint32_swap_big_endian(self): # Issue #23319 class Big(BigEndianStructure): diff --git a/lib-python/3/ctypes/test/test_byteswap.py b/lib-python/3/ctypes/test/test_byteswap.py --- a/lib-python/3/ctypes/test/test_byteswap.py +++ b/lib-python/3/ctypes/test/test_byteswap.py @@ -2,6 +2,7 @@ from binascii import hexlify from ctypes import * +from test.support import impl_detail def bin(s): return hexlify(memoryview(s)).decode().upper() @@ -22,6 +23,7 @@ setattr(bits, "i%s" % i, 1) dump(bits) + @impl_detail("slots are irrelevant on PyPy", pypy=False) def test_slots(self): class BigPoint(BigEndianStructure): __slots__ = () diff --git a/lib-python/3/ctypes/test/test_frombuffer.py b/lib-python/3/ctypes/test/test_frombuffer.py --- a/lib-python/3/ctypes/test/test_frombuffer.py +++ b/lib-python/3/ctypes/test/test_frombuffer.py @@ -85,7 +85,6 @@ del a gc.collect() # Should not crash - @xfail def test_from_buffer_copy(self): a = array.array("i", range(16)) x = (c_int * 16).from_buffer_copy(a) diff --git a/lib-python/3/ctypes/test/test_values.py b/lib-python/3/ctypes/test/test_values.py --- a/lib-python/3/ctypes/test/test_values.py +++ b/lib-python/3/ctypes/test/test_values.py @@ -4,6 +4,7 @@ import unittest import sys +from test.support import cpython_only from ctypes import * import _ctypes_test @@ -28,6 +29,7 @@ ctdll = CDLL(_ctypes_test.__file__) self.assertRaises(ValueError, c_int.in_dll, ctdll, "Undefined_Symbol") + at cpython_only class PythonValuesTestCase(unittest.TestCase): """This test only works when python itself is a dll/shared library""" diff --git a/lib-python/3/doctest.py b/lib-python/3/doctest.py --- a/lib-python/3/doctest.py +++ b/lib-python/3/doctest.py @@ -939,6 +939,8 @@ elif inspect.getmodule(object) is not None: return module is inspect.getmodule(object) elif inspect.isfunction(object): + if isinstance(object.__code__, inspect._builtin_code_type): + return True # XXX: A PyPy builtin - no way to tell return module.__dict__ is object.__globals__ elif inspect.ismethoddescriptor(object): if hasattr(object, '__objclass__'): diff --git a/lib-python/3/idlelib/CallTips.py b/lib-python/3/idlelib/CallTips.py --- a/lib-python/3/idlelib/CallTips.py +++ b/lib-python/3/idlelib/CallTips.py @@ -123,6 +123,15 @@ _first_param = re.compile('(?<=\()\w*\,?\s*') _default_callable_argspec = "See source or doc" +def _is_user_method(ob): + """Detect user methods on PyPy""" + return (isinstance(ob, types.MethodType) and + isinstance(ob.__code__, types.CodeType)) + +def _is_user_function(ob): + """Detect user methods on PyPy""" + return (isinstance(ob, types.FunctionType) and + isinstance(ob.__code__, types.CodeType)) def get_argspec(ob): '''Return a string describing the signature of a callable object, or ''. @@ -140,21 +149,21 @@ return argspec if isinstance(ob, type): fob = ob.__init__ - elif isinstance(ob_call, types.MethodType): + elif _is_user_method(ob_call): fob = ob_call else: fob = ob if (isinstance(fob, (types.FunctionType, types.MethodType)) and hasattr(fob.__code__, 'co_code')): # PyPy: not on argspec = inspect.formatargspec(*inspect.getfullargspec(fob)) - if (isinstance(ob, (type, types.MethodType)) or - isinstance(ob_call, types.MethodType)): + if (_is_user_method(ob) or _is_user_method(ob_call) or + (isinstance(ob, type) and _is_user_function(fob))): argspec = _first_param.sub("", argspec) lines = (textwrap.wrap(argspec, _MAX_COLS, subsequent_indent=_INDENT) if len(argspec) > _MAX_COLS else [argspec] if argspec else []) - if isinstance(ob_call, types.MethodType): + if _is_user_method(ob_call): doc = ob_call.__doc__ else: doc = getattr(ob, "__doc__", "") diff --git a/lib-python/3/idlelib/idle_test/test_calltips.py b/lib-python/3/idlelib/idle_test/test_calltips.py --- a/lib-python/3/idlelib/idle_test/test_calltips.py +++ b/lib-python/3/idlelib/idle_test/test_calltips.py @@ -63,7 +63,7 @@ gtest([].append, append_doc) gtest(List.append, append_doc) - gtest(types.MethodType, "method(function, instance)") + gtest(types.MethodType, "instancemethod(function, instance, class)") gtest(SB(), default_tip) def test_signature_wrap(self): diff --git a/lib-python/3/inspect.py b/lib-python/3/inspect.py --- a/lib-python/3/inspect.py +++ b/lib-python/3/inspect.py @@ -49,6 +49,10 @@ import builtins from operator import attrgetter from collections import namedtuple, OrderedDict +try: + from cpyext import is_cpyext_function as _is_cpyext_function +except ImportError: + _is_cpyext_function = lambda obj: False # Create constants for the compiler flags in Include/code.h # We try to get them from dis to avoid duplication @@ -262,7 +266,7 @@ __doc__ documentation string __name__ original name of this function or method __self__ instance to which a method is bound, or None""" - return isinstance(object, types.BuiltinFunctionType) + return isinstance(object, types.BuiltinFunctionType) or _is_cpyext_function(object) def isroutine(object): """Return true if the object is any kind of function or method.""" @@ -1824,7 +1828,7 @@ kwdefaults = getattr(obj, '__kwdefaults__', _void) # ... and not None here annotations = getattr(obj, '__annotations__', None) - return (isinstance(code, types.CodeType) and + return (isinstance(code, (types.CodeType, _builtin_code_type)) and isinstance(name, str) and (defaults is None or isinstance(defaults, tuple)) and (kwdefaults is None or isinstance(kwdefaults, dict)) and @@ -2078,8 +2082,6 @@ s = getattr(func, "__text_signature__", None) if not s: - if func is object: # XXX PyPy hack until we support __text_signature__ - return '()' # in the same cases as CPython raise ValueError("no signature found for builtin {!r}".format(func)) return _signature_fromstr(cls, func, s, skip_bound_arg) diff --git a/lib-python/3/subprocess.py b/lib-python/3/subprocess.py --- a/lib-python/3/subprocess.py +++ b/lib-python/3/subprocess.py @@ -1560,7 +1560,7 @@ 'copyfile' in caller.f_globals): dest_dir = sys.pypy_resolvedirof(target_executable) src_dir = sys.pypy_resolvedirof(sys.executable) - for libname in ['libpypy3-c.so', 'libpypy3-c.dylib']: + for libname in ['libpypy3-c.so', 'libpypy3-c.dylib', 'libpypy3-c.dll']: dest_library = os.path.join(dest_dir, libname) src_library = os.path.join(src_dir, libname) if os.path.exists(src_library): diff --git a/lib-python/3/test/test_bytes.py b/lib-python/3/test/test_bytes.py --- a/lib-python/3/test/test_bytes.py +++ b/lib-python/3/test/test_bytes.py @@ -721,9 +721,12 @@ self.assertIs(type(BytesSubclass(A())), BytesSubclass) # Test PyBytes_FromFormat() - @test.support.impl_detail("don't test cpyext here") def test_from_format(self): test.support.import_module('ctypes') + try: + from ctypes import pythonapi + except ImportError: + self.skipTest( "no pythonapi in ctypes") from ctypes import pythonapi, py_object, c_int, c_char_p PyBytes_FromFormat = pythonapi.PyBytes_FromFormat PyBytes_FromFormat.restype = py_object diff --git a/lib-python/3/test/test_capi.py b/lib-python/3/test/test_capi.py --- a/lib-python/3/test/test_capi.py +++ b/lib-python/3/test/test_capi.py @@ -29,8 +29,9 @@ skips = [] if support.check_impl_detail(pypy=True): skips += [ - 'test_widechar', - ] + 'test_lazy_hash_inheritance', + 'test_capsule', + ] def testfunction(self): """some doc""" @@ -53,6 +54,8 @@ self.assertEqual(testfunction.attribute, "test") self.assertRaises(AttributeError, setattr, inst.testfunction, "attribute", "test") + @unittest.skipIf(support.check_impl_detail(pypy=True), + "doesn't crash on PyPy") @unittest.skipUnless(threading, 'Threading required for this test.') def test_no_FatalError_infinite_loop(self): with support.SuppressCrashReport(): @@ -205,9 +208,9 @@ else: with self.assertRaises(SystemError) as cm: _testcapi.return_null_without_error() + # PyPy change: different message self.assertRegex(str(cm.exception), - 'return_null_without_error.* ' - 'returned NULL without setting an error') + 'Function returned a NULL result without setting an exception') def test_return_result_with_error(self): # Issue #23571: A function must not return a result with an error set @@ -237,9 +240,9 @@ else: with self.assertRaises(SystemError) as cm: _testcapi.return_result_with_error() + # PyPy change: different message self.assertRegex(str(cm.exception), - 'return_result_with_error.* ' - 'returned a result with an error set') + 'An exception was set, but function returned a value') def test_buildvalue_N(self): _testcapi.test_buildvalue_N() @@ -327,6 +330,8 @@ self.pendingcalls_wait(l, n) + at unittest.skipIf(support.check_impl_detail(pypy=True), + "subinterpreters not implemented on PyPy") class SubinterpreterTest(unittest.TestCase): def test_subinterps(self): diff --git a/lib-python/3/test/test_cmd_line_script.py b/lib-python/3/test/test_cmd_line_script.py --- a/lib-python/3/test/test_cmd_line_script.py +++ b/lib-python/3/test/test_cmd_line_script.py @@ -43,11 +43,7 @@ _loader = __loader__ if __loader__ is BuiltinImporter else type(__loader__) print('__loader__==%a' % _loader) print('__file__==%a' % __file__) -if __cached__ is not None: - # XXX: test_script_compiled on PyPy - assertEqual(__file__, __cached__) - if not __cached__.endswith(('pyc', 'pyo')): - raise AssertionError('has __cached__ but not compiled') +print('__cached__==%a' % __cached__) print('__package__==%r' % __package__) # Check PEP 451 details import os.path @@ -239,9 +235,8 @@ def test_basic_script(self): with support.temp_dir() as script_dir: script_name = _make_test_script(script_dir, 'script') - package = '' if support.check_impl_detail(pypy=True) else None self._check_script(script_name, script_name, script_name, - script_dir, package, + script_dir, None, importlib.machinery.SourceFileLoader) def test_script_compiled(self): @@ -250,9 +245,8 @@ py_compile.compile(script_name, doraise=True) os.remove(script_name) pyc_file = support.make_legacy_pyc(script_name) - package = '' if support.check_impl_detail(pypy=True) else None self._check_script(pyc_file, pyc_file, - pyc_file, script_dir, package, + pyc_file, script_dir, None, importlib.machinery.SourcelessFileLoader) def test_directory(self): diff --git a/lib-python/3/test/test_compile.py b/lib-python/3/test/test_compile.py --- a/lib-python/3/test/test_compile.py +++ b/lib-python/3/test/test_compile.py @@ -524,7 +524,8 @@ with open(fn, "wb") as fp: fp.write(src) res = script_helper.run_python_until_end(fn)[0] - self.assertIn(b"Non-UTF-8", res.err) + # PyPy change: we have a different error here + self.assertIn(b"SyntaxError", res.err) def test_yet_more_evil_still_undecodable(self): # Issue #25388 diff --git a/lib-python/3/test/test_cprofile.py b/lib-python/3/test/test_cprofile.py --- a/lib-python/3/test/test_cprofile.py +++ b/lib-python/3/test/test_cprofile.py @@ -1,7 +1,7 @@ """Test suite for the cProfile module.""" import sys -from test.support import run_unittest, TESTFN, unlink +from test.support import run_unittest, TESTFN, unlink, cpython_only # rip off all interesting stuff from test_profile import cProfile @@ -16,6 +16,7 @@ return _ProfileOutput # Issue 3895. + @cpython_only def test_bad_counter_during_dealloc(self): import _lsprof # Must use a file as StringIO doesn't trigger the bug. diff --git a/lib-python/3/test/test_descr.py b/lib-python/3/test/test_descr.py --- a/lib-python/3/test/test_descr.py +++ b/lib-python/3/test/test_descr.py @@ -4278,7 +4278,10 @@ c = C() c.__dict__[Evil()] = 0 - self.assertEqual(c.attr, 1) + try: + self.assertEqual(c.attr, 1) + except AttributeError: # when Evil.__eq__ is called twice + pass # this makes a crash more likely: support.gc_collect() self.assertNotHasAttr(c, 'attr') diff --git a/lib-python/3/test/test_dis.py b/lib-python/3/test/test_dis.py --- a/lib-python/3/test/test_dis.py +++ b/lib-python/3/test/test_dis.py @@ -146,24 +146,26 @@ 1) pass +# PyPy change: JUMP_IF_NOT_DEBUG dis_bug1333982 = """\ -%3d 0 LOAD_CONST 1 (0) - 3 POP_JUMP_IF_TRUE 35 - 6 LOAD_GLOBAL 0 (AssertionError) - 9 LOAD_CONST 2 ( at 0x..., file "%s", line %d>) - 12 LOAD_CONST 3 ('bug1333982..') - 15 MAKE_FUNCTION 0 - 18 LOAD_FAST 0 (x) - 21 GET_ITER - 22 CALL_FUNCTION 1 (1 positional, 0 keyword pair) +%3d 0 JUMP_IF_NOT_DEBUG 35 (to 38) + 3 LOAD_CONST 1 (0) + 6 POP_JUMP_IF_TRUE 38 + 9 LOAD_GLOBAL 0 (AssertionError) + 12 LOAD_CONST 2 ( at 0x..., file "%s", line %d>) + 15 LOAD_CONST 3 ('bug1333982..') + 18 MAKE_FUNCTION 0 + 21 LOAD_FAST 0 (x) + 24 GET_ITER + 25 CALL_FUNCTION 1 (1 positional, 0 keyword pair) -%3d 25 LOAD_CONST 4 (1) - 28 BINARY_ADD - 29 CALL_FUNCTION 1 (1 positional, 0 keyword pair) - 32 RAISE_VARARGS 1 +%3d 28 LOAD_CONST 4 (1) + 31 BINARY_ADD + 32 CALL_FUNCTION 1 (1 positional, 0 keyword pair) + 35 RAISE_VARARGS 1 -%3d >> 35 LOAD_CONST 0 (None) - 38 RETURN_VALUE +%3d >> 38 LOAD_CONST 0 (None) + 41 RETURN_VALUE """ % (bug1333982.__code__.co_firstlineno + 1, __file__, bug1333982.__code__.co_firstlineno + 1, diff --git a/lib-python/3/test/test_doctest.py b/lib-python/3/test/test_doctest.py --- a/lib-python/3/test/test_doctest.py +++ b/lib-python/3/test/test_doctest.py @@ -660,7 +660,7 @@ >>> import builtins >>> tests = doctest.DocTestFinder().find(builtins) - >>> lo, hi = (120, 140) if is_pypy else (790, 810) + >>> lo, hi = (420, 440) if is_pypy else (790, 810) >>> lo < len(tests) < hi # approximate number of objects with docstrings True >>> real_tests = [t for t in tests if len(t.examples) > 0] diff --git a/lib-python/3/test/test_inspect.py b/lib-python/3/test/test_inspect.py --- a/lib-python/3/test/test_inspect.py +++ b/lib-python/3/test/test_inspect.py @@ -32,6 +32,8 @@ from test.support import check_impl_detail from test.test_import import _ready_to_import +if check_impl_detail(): + import _pickle # Functions tested in this suite: @@ -362,6 +364,7 @@ self.assertEqual(inspect.getdoc(mod.FesteringGob.contradiction), 'The automatic gainsaying.') + @cpython_only # XXX: _finddoc() is broken on PyPy, but getdoc() seems OK @unittest.skipIf(MISSING_C_DOCSTRINGS, "test requires docstrings") def test_finddoc(self): finddoc = inspect._finddoc @@ -755,21 +758,23 @@ @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_getfullargspec_builtin_methods(self): - import _pickle - self.assertFullArgSpecEquals(_pickle.Pickler.dump, - args_e=['self', 'obj'], formatted='(self, obj)') - - self.assertFullArgSpecEquals(_pickle.Pickler(io.BytesIO()).dump, - args_e=['self', 'obj'], formatted='(self, obj)') + if check_impl_detail(): + self.assertFullArgSpecEquals(_pickle.Pickler.dump, + args_e=['self', 'obj'], formatted='(self, obj)') + + self.assertFullArgSpecEquals(_pickle.Pickler(io.BytesIO()).dump, + args_e=['self', 'obj'], formatted='(self, obj)') + + # platform-dependent on PyPy + default_fd = os.stat.__kwdefaults__['dir_fd'] self.assertFullArgSpecEquals( os.stat, args_e=['path'], kwonlyargs_e=['dir_fd', 'follow_symlinks'], - kwonlydefaults_e={'dir_fd': None, 'follow_symlinks': True}, - formatted='(path, *, dir_fd=None, follow_symlinks=True)') - - @cpython_only + kwonlydefaults_e={'dir_fd': default_fd, 'follow_symlinks': True}, + formatted='(path, *, dir_fd={}, follow_symlinks=True)'.format(default_fd)) + @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_getfullagrspec_builtin_func(self): @@ -778,7 +783,6 @@ spec = inspect.getfullargspec(builtin) self.assertEqual(spec.defaults[0], 'avocado') - @cpython_only @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_getfullagrspec_builtin_func_no_signature(self): @@ -816,7 +820,9 @@ attrs = attrs_wo_objs(A) - self.assertIn(('__new__', 'method', object), attrs, 'missing __new__') + # changed in PyPy + self.assertIn(('__new__', 'static method', object), attrs, 'missing __new__') + self.assertIn(('__init__', 'method', object), attrs, 'missing __init__') self.assertIn(('s', 'static method', A), attrs, 'missing static method') @@ -1959,12 +1965,10 @@ ('kwargs', ..., int, "var_keyword")), ...)) - @cpython_only @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_signature_on_builtins(self): import _testcapi - import _pickle def test_unbound_method(o): """Use this to test unbound methods (things that should have a self)""" @@ -1998,9 +2002,10 @@ # normal method # (PyMethodDescr_Type, "method_descriptor") - test_unbound_method(_pickle.Pickler.dump) - d = _pickle.Pickler(io.StringIO()) - test_callable(d.dump) + if check_impl_detail(): + test_unbound_method(_pickle.Pickler.dump) + d = _pickle.Pickler(io.StringIO()) + test_callable(d.dump) # static method test_callable(str.maketrans) @@ -2021,7 +2026,7 @@ # This doesn't work now. # (We don't have a valid signature for "type" in 3.4) - with self.assertRaisesRegex(ValueError, "no signature found"): + with self.assertRaisesRegex(ValueError, "signature"): class ThisWorksNow: __call__ = type test_callable(ThisWorksNow()) @@ -2033,7 +2038,6 @@ # Regression test for issue #20586 test_callable(_testcapi.docstring_with_signature_but_no_doc) - @cpython_only @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_signature_on_decorated_builtins(self): @@ -2056,7 +2060,6 @@ follow_wrapped=False), inspect.signature(wrapper_like)) - @cpython_only def test_signature_on_builtins_no_signature(self): import _testcapi with self.assertRaisesRegex(ValueError, @@ -2632,10 +2635,10 @@ with self.assertRaisesRegex(ValueError, "callable.*is not supported"): self.assertEqual(inspect.signature(D), None) + @cpython_only @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_signature_on_builtin_class(self): - import _pickle self.assertEqual(str(inspect.signature(_pickle.Pickler)), '(file, protocol=None, fix_imports=True)') @@ -2881,10 +2884,10 @@ foo_sig = MySignature.from_callable(foo) self.assertTrue(isinstance(foo_sig, MySignature)) + @cpython_only @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_signature_from_callable_builtin_obj(self): - import _pickle class MySignature(inspect.Signature): pass sig = MySignature.from_callable(_pickle.Pickler) self.assertTrue(isinstance(sig, MySignature)) @@ -3417,7 +3420,6 @@ # This test case provides a home for checking that particular APIs # have signatures available for introspection - @cpython_only @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_builtins_have_signatures(self): diff --git a/lib-python/3/test/test_io.py b/lib-python/3/test/test_io.py --- a/lib-python/3/test/test_io.py +++ b/lib-python/3/test/test_io.py @@ -1169,12 +1169,7 @@ b = bytearray(2*buffer_size) self.assertEqual(bufio.peek(3), b'fgh') self.assertEqual(rawio._reads, 3) - self.assertEqual(bufio.readinto1(b), 6) # fails because of - # an apparent inconsistency in CPython: readinto1(), if the - # buffered amount is smaller, would always issue one raw read() - # call. This differs from read1(), which if the buffered amount - # if smaller (but more than zero), would just return it without - # any raw read() call. In PyPy both have the behavior of read1(). + self.assertEqual(bufio.readinto1(b), 6) self.assertEqual(b[:6], b"fghjkl") self.assertEqual(rawio._reads, 4) diff --git a/lib-python/3/test/test_pydoc.py b/lib-python/3/test/test_pydoc.py --- a/lib-python/3/test/test_pydoc.py +++ b/lib-python/3/test/test_pydoc.py @@ -141,7 +141,7 @@  
Modules - +\x20\x20\x20\x20        
builtins

@@ -878,7 +878,7 @@ @requires_docstrings def test_unbound_builtin_method(self): self.assertEqual(self._get_summary_line(pickle.Pickler.dump), - "dump(self, obj, /)") + "dump(self, obj)") # these no longer include "self" def test_bound_python_method(self): @@ -891,13 +891,13 @@ s = StringIO() p = pickle.Pickler(s) self.assertEqual(self._get_summary_line(p.dump), - "dump(obj, /) method of _pickle.Pickler instance") + "dump(obj) method of pickle._Pickler instance") # this should *never* include self! @requires_docstrings def test_module_level_callable(self): self.assertEqual(self._get_summary_line(os.stat), - "stat(path, *, dir_fd=None, follow_symlinks=True)") + "stat(path, *, dir_fd=-100, follow_symlinks=True)") @unittest.skipUnless(threading, 'Threading required for this test.') diff --git a/lib-python/3/test/test_tracemalloc.py b/lib-python/3/test/test_tracemalloc.py --- a/lib-python/3/test/test_tracemalloc.py +++ b/lib-python/3/test/test_tracemalloc.py @@ -1,7 +1,6 @@ import contextlib import os import sys -import tracemalloc import unittest from unittest.mock import patch from test.support.script_helper import (assert_python_ok, assert_python_failure, @@ -12,6 +11,11 @@ except ImportError: threading = None +try: + import tracemalloc +except ImportError: + raise unittest.SkipTest("tracemalloc is required") + EMPTY_STRING_SIZE = sys.getsizeof(b'') def get_frames(nframe, lineno_delta): diff --git a/lib-python/3/test/test_unicode.py b/lib-python/3/test/test_unicode.py --- a/lib-python/3/test/test_unicode.py +++ b/lib-python/3/test/test_unicode.py @@ -2396,6 +2396,10 @@ # Test PyUnicode_FromFormat() def test_from_format(self): support.import_module('ctypes') + try: + from ctypes import pythonapi + except ImportError: + self.skipTest( "no pythonapi in ctypes") from ctypes import ( pythonapi, py_object, sizeof, c_int, c_long, c_longlong, c_ssize_t, diff --git a/lib-python/3/traceback.py b/lib-python/3/traceback.py --- a/lib-python/3/traceback.py +++ b/lib-python/3/traceback.py @@ -544,8 +544,8 @@ yield ' {}\n'.format(badline.strip()) if offset is not None: caretspace = badline.rstrip('\n') - offset = min(len(caretspace), offset) - 1 - caretspace = caretspace[:offset].lstrip() + # bug in CPython: the case offset==0 is mishandled + caretspace = caretspace[:offset].lstrip()[:-1] # non-space whitespace (likes tabs) must be kept for alignment caretspace = ((c.isspace() and c or ' ') for c in caretspace) yield ' {}^\n'.format(''.join(caretspace)) diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/cryptography.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/cryptography.py --- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/cryptography.py +++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/cryptography.py @@ -48,6 +48,9 @@ #else #define CRYPTOGRAPHY_IS_LIBRESSL 0 #endif + +#define CRYPTOGRAPHY_LIBRESSL_251_OR_GREATER \ + (CRYPTOGRAPHY_IS_LIBRESSL && LIBRESSL_VERSION_NUMBER >= 0x20501000) """ TYPES = """ diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509_vfy.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509_vfy.py --- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509_vfy.py +++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509_vfy.py @@ -244,6 +244,14 @@ static const long X509_V_FLAG_SUITEB_192_LOS = 0; static const long X509_V_FLAG_SUITEB_128_LOS = 0; +#if CRYPTOGRAPHY_LIBRESSL_251_OR_GREATER +int X509_VERIFY_PARAM_set1_host(X509_VERIFY_PARAM *, const char *, size_t); +int X509_VERIFY_PARAM_set1_email(X509_VERIFY_PARAM *, const char *, size_t); +int X509_VERIFY_PARAM_set1_ip(X509_VERIFY_PARAM *, const unsigned char *, + size_t); +int X509_VERIFY_PARAM_set1_ip_asc(X509_VERIFY_PARAM *, const char *); +void X509_VERIFY_PARAM_set_hostflags(X509_VERIFY_PARAM *, unsigned int); +#else int (*X509_VERIFY_PARAM_set1_host)(X509_VERIFY_PARAM *, const char *, size_t) = NULL; int (*X509_VERIFY_PARAM_set1_email)(X509_VERIFY_PARAM *, const char *, @@ -254,6 +262,7 @@ void (*X509_VERIFY_PARAM_set_hostflags)(X509_VERIFY_PARAM *, unsigned int) = NULL; #endif +#endif /* OpenSSL 1.0.2+ or Solaris's backport */ #ifdef X509_V_FLAG_PARTIAL_CHAIN diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py --- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py +++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py @@ -20,9 +20,15 @@ SSL_ERROR_EOF, SSL_ERROR_NO_SOCKET, SSL_ERROR_INVALID_ERROR_CODE, pyerr_write_unraisable) from _cffi_ssl._stdssl import error -from select import poll, POLLIN, POLLOUT, select +from select import select from enum import IntEnum as _IntEnum +if sys.platform == 'win32': + HAVE_POLL = False +else: + from select import poll, POLLIN, POLLOUT + HAVE_POLL = True + OPENSSL_VERSION = ffi.string(lib.OPENSSL_VERSION_TEXT).decode('utf-8') OPENSSL_VERSION_NUMBER = lib.OPENSSL_VERSION_NUMBER ver = OPENSSL_VERSION_NUMBER @@ -158,8 +164,6 @@ def _monotonic_clock(): return time.clock_gettime(time.CLOCK_MONOTONIC) -HAVE_POLL = True - def _ssl_select(sock, writing, timeout): if HAVE_POLL: p = poll() diff --git a/lib_pypy/_cffi_ssl/osx-roots.diff b/lib_pypy/_cffi_ssl/osx-roots.diff new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/osx-roots.diff @@ -0,0 +1,475 @@ +diff -Naur libressl-2.6.2.orig/crypto/Makefile.am libressl-2.6.2/crypto/Makefile.am +--- libressl-2.6.2.orig/crypto/Makefile.am 2017-09-02 01:49:55.000000000 +0200 ++++ libressl-2.6.2/crypto/Makefile.am 2017-10-07 14:05:16.000000000 +0200 +@@ -92,7 +92,7 @@ + -mv crypto_portable.sym.tmp crypto_portable.sym + endif + +-libcrypto_la_LDFLAGS = -version-info @LIBCRYPTO_VERSION@ -no-undefined -export-symbols crypto_portable.sym ++libcrypto_la_LDFLAGS = -version-info @LIBCRYPTO_VERSION@ -no-undefined -export-symbols crypto_portable.sym -framework Security -framework CoreFoundation + libcrypto_la_LIBADD = libcompat.la + if !HAVE_EXPLICIT_BZERO + libcrypto_la_LIBADD += libcompatnoopt.la +@@ -863,6 +863,7 @@ + libcrypto_la_SOURCES += x509/x509_txt.c + libcrypto_la_SOURCES += x509/x509_v3.c + libcrypto_la_SOURCES += x509/x509_vfy.c ++libcrypto_la_SOURCES += x509/x509_vfy_apple.c + libcrypto_la_SOURCES += x509/x509_vpm.c + libcrypto_la_SOURCES += x509/x509cset.c + libcrypto_la_SOURCES += x509/x509name.c +diff -Naur libressl-2.6.2.orig/crypto/Makefile.in libressl-2.6.2/crypto/Makefile.in +--- libressl-2.6.2.orig/crypto/Makefile.in 2017-09-26 06:07:03.000000000 +0200 ++++ libressl-2.6.2/crypto/Makefile.in 2017-10-07 14:05:24.000000000 +0200 +@@ -426,20 +426,20 @@ + x509/x509_err.c x509/x509_ext.c x509/x509_lu.c x509/x509_obj.c \ + x509/x509_r2x.c x509/x509_req.c x509/x509_set.c \ + x509/x509_trs.c x509/x509_txt.c x509/x509_v3.c x509/x509_vfy.c \ +- x509/x509_vpm.c x509/x509cset.c x509/x509name.c \ +- x509/x509rset.c x509/x509spki.c x509/x509type.c x509/x_all.c \ +- x509v3/pcy_cache.c x509v3/pcy_data.c x509v3/pcy_lib.c \ +- x509v3/pcy_map.c x509v3/pcy_node.c x509v3/pcy_tree.c \ +- x509v3/v3_akey.c x509v3/v3_akeya.c x509v3/v3_alt.c \ +- x509v3/v3_bcons.c x509v3/v3_bitst.c x509v3/v3_conf.c \ +- x509v3/v3_cpols.c x509v3/v3_crld.c x509v3/v3_enum.c \ +- x509v3/v3_extku.c x509v3/v3_genn.c x509v3/v3_ia5.c \ +- x509v3/v3_info.c x509v3/v3_int.c x509v3/v3_lib.c \ +- x509v3/v3_ncons.c x509v3/v3_ocsp.c x509v3/v3_pci.c \ +- x509v3/v3_pcia.c x509v3/v3_pcons.c x509v3/v3_pku.c \ +- x509v3/v3_pmaps.c x509v3/v3_prn.c x509v3/v3_purp.c \ +- x509v3/v3_skey.c x509v3/v3_sxnet.c x509v3/v3_utl.c \ +- x509v3/v3err.c ++ x509/x509_vfy_apple.c x509/x509_vpm.c x509/x509cset.c \ ++ x509/x509name.c x509/x509rset.c x509/x509spki.c \ ++ x509/x509type.c x509/x_all.c x509v3/pcy_cache.c \ ++ x509v3/pcy_data.c x509v3/pcy_lib.c x509v3/pcy_map.c \ ++ x509v3/pcy_node.c x509v3/pcy_tree.c x509v3/v3_akey.c \ ++ x509v3/v3_akeya.c x509v3/v3_alt.c x509v3/v3_bcons.c \ ++ x509v3/v3_bitst.c x509v3/v3_conf.c x509v3/v3_cpols.c \ ++ x509v3/v3_crld.c x509v3/v3_enum.c x509v3/v3_extku.c \ ++ x509v3/v3_genn.c x509v3/v3_ia5.c x509v3/v3_info.c \ ++ x509v3/v3_int.c x509v3/v3_lib.c x509v3/v3_ncons.c \ ++ x509v3/v3_ocsp.c x509v3/v3_pci.c x509v3/v3_pcia.c \ ++ x509v3/v3_pcons.c x509v3/v3_pku.c x509v3/v3_pmaps.c \ ++ x509v3/v3_prn.c x509v3/v3_purp.c x509v3/v3_skey.c \ ++ x509v3/v3_sxnet.c x509v3/v3_utl.c x509v3/v3err.c + am__objects_27 = aes/libcrypto_la-aes-elf-x86_64.lo \ + aes/libcrypto_la-bsaes-elf-x86_64.lo \ + aes/libcrypto_la-vpaes-elf-x86_64.lo \ +@@ -759,11 +759,12 @@ + x509/libcrypto_la-x509_r2x.lo x509/libcrypto_la-x509_req.lo \ + x509/libcrypto_la-x509_set.lo x509/libcrypto_la-x509_trs.lo \ + x509/libcrypto_la-x509_txt.lo x509/libcrypto_la-x509_v3.lo \ +- x509/libcrypto_la-x509_vfy.lo x509/libcrypto_la-x509_vpm.lo \ +- x509/libcrypto_la-x509cset.lo x509/libcrypto_la-x509name.lo \ +- x509/libcrypto_la-x509rset.lo x509/libcrypto_la-x509spki.lo \ +- x509/libcrypto_la-x509type.lo x509/libcrypto_la-x_all.lo \ +- x509v3/libcrypto_la-pcy_cache.lo \ ++ x509/libcrypto_la-x509_vfy.lo \ ++ x509/libcrypto_la-x509_vfy_apple.lo \ ++ x509/libcrypto_la-x509_vpm.lo x509/libcrypto_la-x509cset.lo \ ++ x509/libcrypto_la-x509name.lo x509/libcrypto_la-x509rset.lo \ ++ x509/libcrypto_la-x509spki.lo x509/libcrypto_la-x509type.lo \ ++ x509/libcrypto_la-x_all.lo x509v3/libcrypto_la-pcy_cache.lo \ + x509v3/libcrypto_la-pcy_data.lo x509v3/libcrypto_la-pcy_lib.lo \ + x509v3/libcrypto_la-pcy_map.lo x509v3/libcrypto_la-pcy_node.lo \ + x509v3/libcrypto_la-pcy_tree.lo x509v3/libcrypto_la-v3_akey.lo \ +@@ -1000,7 +1001,7 @@ + $(ASM_X86_64_ELF) $(ASM_X86_64_MACOSX) + BUILT_SOURCES = crypto_portable.sym + CLEANFILES = crypto_portable.sym +-libcrypto_la_LDFLAGS = -version-info @LIBCRYPTO_VERSION@ -no-undefined -export-symbols crypto_portable.sym ++libcrypto_la_LDFLAGS = -version-info @LIBCRYPTO_VERSION@ -no-undefined -export-symbols crypto_portable.sym -framework Security -framework CoreFoundation + libcrypto_la_LIBADD = libcompat.la $(am__append_1) + libcrypto_la_CPPFLAGS = $(AM_CPPFLAGS) -DLIBRESSL_INTERNAL \ + -DOPENSSL_NO_HW_PADLOCK $(am__append_2) $(am__append_3) \ +@@ -1272,20 +1273,20 @@ + x509/x509_err.c x509/x509_ext.c x509/x509_lu.c x509/x509_obj.c \ + x509/x509_r2x.c x509/x509_req.c x509/x509_set.c \ + x509/x509_trs.c x509/x509_txt.c x509/x509_v3.c x509/x509_vfy.c \ +- x509/x509_vpm.c x509/x509cset.c x509/x509name.c \ +- x509/x509rset.c x509/x509spki.c x509/x509type.c x509/x_all.c \ +- x509v3/pcy_cache.c x509v3/pcy_data.c x509v3/pcy_lib.c \ +- x509v3/pcy_map.c x509v3/pcy_node.c x509v3/pcy_tree.c \ +- x509v3/v3_akey.c x509v3/v3_akeya.c x509v3/v3_alt.c \ +- x509v3/v3_bcons.c x509v3/v3_bitst.c x509v3/v3_conf.c \ +- x509v3/v3_cpols.c x509v3/v3_crld.c x509v3/v3_enum.c \ +- x509v3/v3_extku.c x509v3/v3_genn.c x509v3/v3_ia5.c \ +- x509v3/v3_info.c x509v3/v3_int.c x509v3/v3_lib.c \ +- x509v3/v3_ncons.c x509v3/v3_ocsp.c x509v3/v3_pci.c \ +- x509v3/v3_pcia.c x509v3/v3_pcons.c x509v3/v3_pku.c \ +- x509v3/v3_pmaps.c x509v3/v3_prn.c x509v3/v3_purp.c \ +- x509v3/v3_skey.c x509v3/v3_sxnet.c x509v3/v3_utl.c \ +- x509v3/v3err.c ++ x509/x509_vfy_apple.c x509/x509_vpm.c x509/x509cset.c \ ++ x509/x509name.c x509/x509rset.c x509/x509spki.c \ ++ x509/x509type.c x509/x_all.c x509v3/pcy_cache.c \ ++ x509v3/pcy_data.c x509v3/pcy_lib.c x509v3/pcy_map.c \ ++ x509v3/pcy_node.c x509v3/pcy_tree.c x509v3/v3_akey.c \ ++ x509v3/v3_akeya.c x509v3/v3_alt.c x509v3/v3_bcons.c \ ++ x509v3/v3_bitst.c x509v3/v3_conf.c x509v3/v3_cpols.c \ ++ x509v3/v3_crld.c x509v3/v3_enum.c x509v3/v3_extku.c \ ++ x509v3/v3_genn.c x509v3/v3_ia5.c x509v3/v3_info.c \ ++ x509v3/v3_int.c x509v3/v3_lib.c x509v3/v3_ncons.c \ ++ x509v3/v3_ocsp.c x509v3/v3_pci.c x509v3/v3_pcia.c \ ++ x509v3/v3_pcons.c x509v3/v3_pku.c x509v3/v3_pmaps.c \ ++ x509v3/v3_prn.c x509v3/v3_purp.c x509v3/v3_skey.c \ ++ x509v3/v3_sxnet.c x509v3/v3_utl.c x509v3/v3err.c + + # chacha + +@@ -2808,6 +2809,8 @@ + x509/$(DEPDIR)/$(am__dirstamp) + x509/libcrypto_la-x509_vfy.lo: x509/$(am__dirstamp) \ + x509/$(DEPDIR)/$(am__dirstamp) ++x509/libcrypto_la-x509_vfy_apple.lo: x509/$(am__dirstamp) \ ++ x509/$(DEPDIR)/$(am__dirstamp) + x509/libcrypto_la-x509_vpm.lo: x509/$(am__dirstamp) \ + x509/$(DEPDIR)/$(am__dirstamp) + x509/libcrypto_la-x509cset.lo: x509/$(am__dirstamp) \ +@@ -3583,6 +3586,7 @@ + @AMDEP_TRUE@@am__include@ @am__quote at x509/$(DEPDIR)/libcrypto_la-x509_txt.Plo at am__quote@ + @AMDEP_TRUE@@am__include@ @am__quote at x509/$(DEPDIR)/libcrypto_la-x509_v3.Plo at am__quote@ + @AMDEP_TRUE@@am__include@ @am__quote at x509/$(DEPDIR)/libcrypto_la-x509_vfy.Plo at am__quote@ ++ at AMDEP_TRUE@@am__include@ @am__quote at x509/$(DEPDIR)/libcrypto_la-x509_vfy_apple.Plo at am__quote@ + @AMDEP_TRUE@@am__include@ @am__quote at x509/$(DEPDIR)/libcrypto_la-x509_vpm.Plo at am__quote@ + @AMDEP_TRUE@@am__include@ @am__quote at x509/$(DEPDIR)/libcrypto_la-x509cset.Plo at am__quote@ + @AMDEP_TRUE@@am__include@ @am__quote at x509/$(DEPDIR)/libcrypto_la-x509name.Plo at am__quote@ +@@ -7460,6 +7464,13 @@ + @AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ + @am__fastdepCC_FALSE@ $(AM_V_CC at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libcrypto_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o x509/libcrypto_la-x509_vfy.lo `test -f 'x509/x509_vfy.c' || echo '$(srcdir)/'`x509/x509_vfy.c + ++x509/libcrypto_la-x509_vfy_apple.lo: x509/x509_vfy_apple.c ++ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libcrypto_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT x509/libcrypto_la-x509_vfy_apple.lo -MD -MP -MF x509/$(DEPDIR)/libcrypto_la-x509_vfy_apple.Tpo -c -o x509/libcrypto_la-x509_vfy_apple.lo `test -f 'x509/x509_vfy_apple.c' || echo '$(srcdir)/'`x509/x509_vfy_apple.c ++ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) x509/$(DEPDIR)/libcrypto_la-x509_vfy_apple.Tpo x509/$(DEPDIR)/libcrypto_la-x509_vfy_apple.Plo ++ at AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='x509/x509_vfy_apple.c' object='x509/libcrypto_la-x509_vfy_apple.lo' libtool=yes @AMDEPBACKSLASH@ ++ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ ++ at am__fastdepCC_FALSE@ $(AM_V_CC at am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libcrypto_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o x509/libcrypto_la-x509_vfy_apple.lo `test -f 'x509/x509_vfy_apple.c' || echo '$(srcdir)/'`x509/x509_vfy_apple.c ++ + x509/libcrypto_la-x509_vpm.lo: x509/x509_vpm.c + @am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libcrypto_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT x509/libcrypto_la-x509_vpm.lo -MD -MP -MF x509/$(DEPDIR)/libcrypto_la-x509_vpm.Tpo -c -o x509/libcrypto_la-x509_vpm.lo `test -f 'x509/x509_vpm.c' || echo '$(srcdir)/'`x509/x509_vpm.c + @am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) x509/$(DEPDIR)/libcrypto_la-x509_vpm.Tpo x509/$(DEPDIR)/libcrypto_la-x509_vpm.Plo +diff -Naur libressl-2.6.2.orig/crypto/x509/x509_vfy.c libressl-2.6.2/crypto/x509/x509_vfy.c +--- libressl-2.6.2.orig/crypto/x509/x509_vfy.c 2017-09-02 14:01:08.000000000 +0200 ++++ libressl-2.6.2/crypto/x509/x509_vfy.c 2017-10-07 14:05:16.000000000 +0200 +@@ -115,6 +115,13 @@ + + #define CRL_SCORE_TIME_DELTA 0x002 + ++/* ++ * If we are using Trust Evaluation Agent, rename the original function ++ */ ++#ifdef __APPLE__ ++#define X509_verify_cert X509_verify_cert_orig ++#endif ++ + static int null_callback(int ok, X509_STORE_CTX *e); + static int check_issued(X509_STORE_CTX *ctx, X509 *x, X509 *issuer); + static X509 *find_issuer(X509_STORE_CTX *ctx, STACK_OF(X509) *sk, X509 *x); +diff -Naur libressl-2.6.2.orig/crypto/x509/x509_vfy_apple.c libressl-2.6.2/crypto/x509/x509_vfy_apple.c +--- libressl-2.6.2.orig/crypto/x509/x509_vfy_apple.c 1970-01-01 01:00:00.000000000 +0100 ++++ libressl-2.6.2/crypto/x509/x509_vfy_apple.c 2017-10-07 14:05:16.000000000 +0200 +@@ -0,0 +1,225 @@ ++/* ++ * Copyright (c) 2009 Apple Inc. All Rights Reserved. ++ * ++ * @APPLE_LICENSE_HEADER_START@ ++ * ++ * This file contains Original Code and/or Modifications of Original Code ++ * as defined in and that are subject to the Apple Public Source License ++ * Version 2.0 (the 'License'). You may not use this file except in ++ * compliance with the License. Please obtain a copy of the License at ++ * http://www.opensource.apple.com/apsl/ and read it before using this ++ * file. ++ * ++ * The Original Code and all software distributed under the License are ++ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER ++ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, ++ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. ++ * Please see the License for the specific language governing rights and ++ * limitations under the License. ++ * ++ * @APPLE_LICENSE_HEADER_END@ ++ * ++ */ ++ ++#include ++#include ++#include ++ ++#include ++ ++#include ++ ++#include ++#include ++#include ++ ++#include "cryptlib.h" ++#include "vpm_int.h" ++#include "x509_vfy_apple.h" ++ ++#define TEA_might_correct_error(err) (err == X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY || err == X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT || err == X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN) ++ ++ ++static bool add_cert_to_array(CFMutableArrayRef array, X509 *x509) ++{ ++ unsigned char *asn1_cert_data = NULL; ++ int asn1_cert_len = i2d_X509(x509, &asn1_cert_data); ++ ++ CFDataRef data = CFDataCreate(kCFAllocatorDefault, asn1_cert_data, asn1_cert_len); ++ ++ if (data == NULL) { ++ return false; ++ } ++ ++ SecCertificateRef cert = SecCertificateCreateWithData(NULL, data); ++ ++ free(asn1_cert_data); ++ ++ if (cert == NULL) { ++ CFRelease(data); ++ return false; ++ } ++ ++ CFArrayAppendValue(array, cert); ++ CFRelease(data); ++ ++ return true; ++} ++ ++static CFStringRef to_string(const char *s) { ++ if (s == NULL) ++ return NULL; ++ return CFStringCreateWithCString(kCFAllocatorDefault, s, ++ kCFStringEncodingASCII); ++} ++ ++static SecPolicyRef get_policy(X509_VERIFY_PARAM *param) { ++ switch (param->purpose) { ++ case X509_PURPOSE_SSL_CLIENT: ++ case X509_PURPOSE_SSL_SERVER: { ++ ++ if (!param->id) { ++ fprintf(stderr, "got no ID!\n"); ++ return NULL; ++ } ++ ++ CFStringRef hostname; ++ int nhosts = sk_OPENSSL_STRING_num(param->id->hosts); ++ ++ if (nhosts != 1) { ++ hostname = NULL; ++ ++ } else { ++ hostname = to_string(sk_OPENSSL_STRING_value(param->id->hosts, 0)); ++ CFShow(hostname); ++ } ++ ++ return SecPolicyCreateSSL(param->purpose == X509_PURPOSE_SSL_SERVER, ++ hostname); ++ } ++ ++ case X509_PURPOSE_NS_SSL_SERVER: ++ case X509_PURPOSE_SMIME_SIGN: ++ case X509_PURPOSE_SMIME_ENCRYPT: ++ case X509_PURPOSE_CRL_SIGN: ++ case X509_PURPOSE_ANY: ++ case X509_PURPOSE_OCSP_HELPER: ++ case X509_PURPOSE_TIMESTAMP_SIGN: ++ default: ++ fprintf(stderr, "unsupported purpose %d", param->purpose); ++ return NULL; ++ } ++} ++ ++/* ++ * Please see comment in x509_vfy_apple.h ++ */ ++int ++X509_verify_cert(X509_STORE_CTX *ctx) ++{ ++ uint64_t certLastIndex = 0; ++ uint64_t i = 0; ++ ++ /* Try OpenSSL, if we get a local certificate issue verify against trusted roots */ ++ int ret = X509_verify_cert_orig(ctx); ++ ++ /* Verify TEA is enabled and should be used. */ ++ if (0 == X509_TEA_is_enabled() || ++ ret == 1 || !TEA_might_correct_error(ctx->error)) { ++ return ret; ++ } ++ ++ /* Verify that the certificate chain exists, otherwise make it. */ ++ if (ctx->chain == NULL && (ctx->chain = sk_X509_new_null()) == NULL) { ++ fprintf(stderr, "Could not create the certificate chain"); ++ ctx->error = X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY; ++ return -1; ++ } ++ ++ /* Verify chain depth */ ++ certLastIndex = sk_X509_num(ctx->untrusted); ++ if (certLastIndex > ctx->param->depth) { ++ fprintf(stderr, "Pruning certificate chain to %" PRIu64, certLastIndex); ++ certLastIndex = ctx->param->depth; ++ } ++ ++ CFMutableArrayRef certArray = CFArrayCreateMutable(NULL, certLastIndex + 1, NULL); ++ CFRetain(certArray); ++ ++ if (!add_cert_to_array(certArray, ctx->cert)) { ++ fprintf(stderr, "Failed to add certificate to array"); ++ CFRelease(certArray); ++ ctx->error = X509_V_ERR_UNSPECIFIED; ++ return -1; ++ } ++ ++ for (i = 0; i < certLastIndex; ++i) { ++ X509 *t = sk_X509_value(ctx->untrusted, i); ++ if (!add_cert_to_array(certArray, t)) { ++ fprintf(stderr, "Failed to add chain certificate %lld to array", i); ++ CFRelease(certArray); ++ ctx->error = X509_V_ERR_UNSPECIFIED; ++ return 0; ++ } ++ } ++ ++ // We put ASN.1 encoded X509 on the CertificateChain, so we don't call TEACertificateChainSetEncodingHandler ++ SecPolicyRef policy = get_policy(ctx->param); ++ ++ if (policy == NULL) { ++ fprintf(stderr, "Failed to create policy!\n"); ++ CFRelease(certArray); ++ ctx->error = X509_V_ERR_UNSPECIFIED; ++ return -1; ++ } ++ ++ SecTrustRef trust = NULL; ++ ++ if (SecTrustCreateWithCertificates(certArray, policy, &trust) != errSecSuccess) { ++ fprintf(stderr, "Failed to create trust!\n"); ++ CFRelease(certArray); ++ ctx->error = X509_V_ERR_CERT_UNTRUSTED; ++ return -1; ++ } ++ ++ if (ctx->param->flags & X509_V_FLAG_USE_CHECK_TIME) { ++ fprintf(stderr, "Setting time not supported yet?\n"); ++ SecTrustSetVerifyDate(trust, CFDateCreate(NULL, ctx->param->check_time)); ++ } ++ ++ SecTrustResultType result = 0; ++ ++ if (SecTrustEvaluate(trust, &result) != errSecSuccess || result != kSecTrustResultUnspecified) { ++ CFRelease(certArray); ++ ctx->error = X509_V_ERR_CERT_UNTRUSTED; ++ return 0; ++ } ++ ++ CFRelease(certArray); ++ ctx->error = 0; ++ return 1; ++} ++ ++#pragma mark Trust Evaluation Agent ++ ++/* -1: not set ++ * 0: set to false ++ * 1: set to true ++ */ ++static int tea_enabled = -1; ++ ++void ++X509_TEA_set_state(int change) ++{ ++ tea_enabled = (change) ? 1 : 0; ++} ++ ++int ++X509_TEA_is_enabled() ++{ ++ if (tea_enabled < 0) ++ tea_enabled = (NULL == getenv(X509_TEA_ENV_DISABLE)); ++ ++ return tea_enabled != 0; ++} +diff -Naur libressl-2.6.2.orig/crypto/x509/x509_vfy_apple.h libressl-2.6.2/crypto/x509/x509_vfy_apple.h +--- libressl-2.6.2.orig/crypto/x509/x509_vfy_apple.h 1970-01-01 01:00:00.000000000 +0100 ++++ libressl-2.6.2/crypto/x509/x509_vfy_apple.h 2017-10-07 14:05:16.000000000 +0200 +@@ -0,0 +1,74 @@ ++/* ++ * Copyright (c) 2009 Apple Inc. All Rights Reserved. ++ * ++ * @APPLE_LICENSE_HEADER_START@ ++ * ++ * This file contains Original Code and/or Modifications of Original Code ++ * as defined in and that are subject to the Apple Public Source License ++ * Version 2.0 (the 'License'). You may not use this file except in ++ * compliance with the License. Please obtain a copy of the License at ++ * http://www.opensource.apple.com/apsl/ and read it before using this ++ * file. ++ * ++ * The Original Code and all software distributed under the License are ++ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER ++ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, ++ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. ++ * Please see the License for the specific language governing rights and ++ * limitations under the License. ++ * ++ * @APPLE_LICENSE_HEADER_END@ ++ * ++ */ ++ ++#ifndef HEADER_X509_H ++#include ++#endif ++ ++#ifndef HEADER_X509_VFY_APPLE_H ++#define HEADER_X509_VFY_APPLE_H ++ ++/* Environment variable name to disable TEA. */ ++#define X509_TEA_ENV_DISABLE "OPENSSL_X509_TEA_DISABLE" ++ ++/* ++ * X509_verify_cert ++ * ++ * Originally located in x509_vfy.c. ++ * ++ * Verify certificate with OpenSSL created X509_verify_cert. If and only if ++ * OpenSSL cannot get certificate issuer locally then OS X security API will ++ * verify the certificate, using Trust Evaluation Agent. ++ * ++ * Return values: ++ * -------------- ++ * -1: Null was passed for either ctx or ctx->cert. ++ * 0: Certificate is trusted. ++ * 1: Certificate is not trusted. ++ */ ++int X509_verify_cert(X509_STORE_CTX *ctx); ++ ++/* ++ * X509_TEA_is_enabled ++ * ++ * Is the Trust Evaluation Agent (TEA) used for certificate verification when ++ * the issuer cannot be verified. ++ * ++ * Returns 0 if TEA is disabled and 1 if TEA is enabled. ++ */ ++int X509_TEA_is_enabled(); ++ ++/* ++ * X509_TEA_set_state ++ * ++ * Enables/disables certificate verification with Trust Evaluation Agent (TEA) ++ * when the issuer cannot be verified. ++ * ++ * Pass 0 to disable TEA and non-zero to enable TEA. ++ */ ++void X509_TEA_set_state(int change); ++ ++int X509_verify_cert_orig(X509_STORE_CTX *ctx); ++ ++#endif /* HEADER_X509_VFY_APPLE_H */ diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py --- a/lib_pypy/_ctypes/array.py +++ b/lib_pypy/_ctypes/array.py @@ -8,9 +8,14 @@ class ArrayMeta(_CDataMeta): def __new__(self, name, cls, typedict): res = type.__new__(self, name, cls, typedict) + if cls == (_CData,): # this is the Array class defined below + res._ffiarray = None return res - + if not hasattr(res, '_length_') or not isinstance(res._length_, int): + raise AttributeError( + "class must define a '_length_' attribute, " + "which must be a positive integer") ffiarray = res._ffiarray = _rawffi.Array(res._type_._ffishape_) subletter = getattr(res._type_, '_type_', None) if subletter == 'c': @@ -55,7 +60,7 @@ for i in range(len(val)): target[i] = val[i] if len(val) < self._length_: - target[len(val)] = '\x00' + target[len(val)] = u'\x00' res.value = property(getvalue, setvalue) res._ffishape_ = (ffiarray, res._length_) @@ -164,7 +169,7 @@ if letter == 'c': return b"".join(l) if letter == 'u': - return "".join(l) + return u"".join(l) return l class Array(_CData, metaclass=ArrayMeta): diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py --- a/lib_pypy/_ctypes/basics.py +++ b/lib_pypy/_ctypes/basics.py @@ -165,6 +165,10 @@ def _get_buffer_value(self): return self._buffer[0] + def _copy_to(self, addr): + target = type(self).from_address(addr)._buffer + target[0] = self._get_buffer_value() + def _to_ffi_param(self): if self.__class__._is_pointer_like(): return self._get_buffer_value() diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py --- a/lib_pypy/_ctypes/pointer.py +++ b/lib_pypy/_ctypes/pointer.py @@ -113,7 +113,9 @@ cobj = self._type_.from_param(value) if ensure_objects(cobj) is not None: store_reference(self, index, cobj._objects) - self._subarray(index)[0] = cobj._get_buffer_value() + address = self._buffer[0] + address += index * sizeof(self._type_) + cobj._copy_to(address) def __bool__(self): return self._buffer[0] != 0 diff --git a/lib_pypy/_ctypes/primitive.py b/lib_pypy/_ctypes/primitive.py --- a/lib_pypy/_ctypes/primitive.py +++ b/lib_pypy/_ctypes/primitive.py @@ -232,9 +232,6 @@ elif tp == 'u': def _setvalue(self, val): - if isinstance(val, bytes): - val = val.decode(ConvMode.encoding, ConvMode.errors) - # possible if we use 'ignore' if val: self._buffer[0] = val def _getvalue(self): @@ -243,8 +240,6 @@ elif tp == 'c': def _setvalue(self, val): - if isinstance(val, str): - val = val.encode(ConvMode.encoding, ConvMode.errors) if val: self._buffer[0] = val def _getvalue(self): diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -290,6 +290,11 @@ def _get_buffer_value(self): return self._buffer.buffer + def _copy_to(self, addr): + from ctypes import memmove + origin = self._get_buffer_value() + memmove(addr, origin, self._fficompositesize_) + def _to_ffi_param(self): return self._buffer diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py --- a/lib_pypy/_ctypes_test.py +++ b/lib_pypy/_ctypes_test.py @@ -21,5 +21,11 @@ with fp: imp.load_module('_ctypes_test', fp, filename, description) except ImportError: + if os.name == 'nt': + # hack around finding compilers on win32 + try: + import setuptools + except ImportError: + pass print('could not find _ctypes_test in %s' % output_dir) _pypy_testcapi.compile_shared('_ctypes_test.c', '_ctypes_test', output_dir) diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py --- a/lib_pypy/_curses.py +++ b/lib_pypy/_curses.py @@ -404,7 +404,7 @@ return val def get_wch(self, *args): - wch = ffi.new("int[1]") + wch = ffi.new("wint_t[1]") if len(args) == 0: val = lib.wget_wch(self._win, wch) elif len(args) == 2: diff --git a/lib_pypy/_pypy_testcapi.py b/lib_pypy/_pypy_testcapi.py --- a/lib_pypy/_pypy_testcapi.py +++ b/lib_pypy/_pypy_testcapi.py @@ -8,7 +8,8 @@ content = fid.read() # from cffi's Verifier() key = '\x00'.join([sys.version[:3], content]) - key += 'cpyext-gc-support-2' # this branch requires recompilation! + # change the key to force recompilation + key += '2017-11-21' if sys.version_info >= (3,): key = key.encode('utf-8') k1 = hex(binascii.crc32(key[0::2]) & 0xffffffff) diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -1080,21 +1080,25 @@ if '\0' in sql: raise ValueError("the query contains a null character") - first_word = sql.lstrip().split(" ")[0].upper() - if first_word == "": + + if sql: + first_word = sql.lstrip().split()[0].upper() + if first_word == '': + self._type = _STMT_TYPE_INVALID + if first_word == "SELECT": + self._type = _STMT_TYPE_SELECT + elif first_word == "INSERT": + self._type = _STMT_TYPE_INSERT + elif first_word == "UPDATE": + self._type = _STMT_TYPE_UPDATE + elif first_word == "DELETE": + self._type = _STMT_TYPE_DELETE + elif first_word == "REPLACE": + self._type = _STMT_TYPE_REPLACE + else: + self._type = _STMT_TYPE_OTHER + else: self._type = _STMT_TYPE_INVALID - elif first_word == "SELECT": - self._type = _STMT_TYPE_SELECT - elif first_word == "INSERT": - self._type = _STMT_TYPE_INSERT - elif first_word == "UPDATE": - self._type = _STMT_TYPE_UPDATE - elif first_word == "DELETE": - self._type = _STMT_TYPE_DELETE - elif first_word == "REPLACE": - self._type = _STMT_TYPE_REPLACE - else: - self._type = _STMT_TYPE_OTHER if isinstance(sql, unicode): sql = sql.encode('utf-8') diff --git a/lib_pypy/_ssl/__init__.py b/lib_pypy/_ssl/__init__.py --- a/lib_pypy/_ssl/__init__.py +++ b/lib_pypy/_ssl/__init__.py @@ -14,3 +14,14 @@ # RAND_egd is optional and might not be available on e.g. libressl if hasattr(_stdssl, 'RAND_egd'): RAND_egd = builtinify(RAND_egd) + +import sys +if sys.platform == "win32" and 'enum_certificates' not in globals(): + def enum_certificates(*args, **kwds): + import warnings + warnings.warn("ssl.enum_certificates() is not implemented") + return [] + def enum_crls(*args, **kwds): + import warnings + warnings.warn("ssl.enum_crls() is not implemented") + return [] diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py --- a/lib_pypy/_testcapi.py +++ b/lib_pypy/_testcapi.py @@ -17,6 +17,12 @@ with fp: imp.load_module('_testcapi', fp, filename, description) except ImportError: + if os.name == 'nt': + # hack around finding compilers on win32 + try: + import setuptools + except ImportError: + pass _pypy_testcapi.compile_shared(cfile, '_testcapi', output_dir) diff --git a/lib_pypy/_testcapimodule.c b/lib_pypy/_testcapimodule.c --- a/lib_pypy/_testcapimodule.c +++ b/lib_pypy/_testcapimodule.c @@ -915,12 +915,6 @@ return -1; } Py_DECREF(res); - if (Py_REFCNT(arg) != 1) { - PyErr_Format(TestError, "test_buildvalue_N: " - "arg was not decrefed in successful " - "Py_BuildValue(\"%s\")", fmt); - return -1; - } From pypy.commits at gmail.com Fri Jan 12 13:59:37 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 12 Jan 2018 10:59:37 -0800 (PST) Subject: [pypy-commit] pypy py3.5-refactor-slots: hg merge py3.5 Message-ID: <5a590599.b6a6df0a.9e35d.0fe5@mx.google.com> Author: Ronan Lamy Branch: py3.5-refactor-slots Changeset: r93661:8b52728662cb Date: 2018-01-12 17:38 +0000 http://bitbucket.org/pypy/pypy/changeset/8b52728662cb/ Log: hg merge py3.5 diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -50,3 +50,4 @@ 0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0 0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0 09f9160b643e3f02ccb8c843b2fbb4e5cbf54082 release-pypy3.5-v5.10.0 +3f6eaa010fce78cc7973bdc1dfdb95970f08fed2 release-pypy3.5-v5.10.1 diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.11.2 +Version: 1.11.3 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.11.2" -__version_info__ = (1, 11, 2) +__version__ = "1.11.3" +__version_info__ = (1, 11, 3) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h --- a/lib_pypy/cffi/_cffi_include.h +++ b/lib_pypy/cffi/_cffi_include.h @@ -7,11 +7,38 @@ we can learn about Py_DEBUG from pyconfig.h, but it is unclear if the same works for the other two macros. Py_DEBUG implies them, but not the other way around. + + Issue #350: more mess: on Windows, with _MSC_VER, we have to define + Py_LIMITED_API even before including pyconfig.h. In that case, we + guess what pyconfig.h will do to the macros above, and check our + guess after the #include. */ #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) -# include -# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) -# define Py_LIMITED_API +# ifdef _MSC_VER +# if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API +# endif +# include + /* sanity-check: Py_LIMITED_API will cause crashes if any of these + are also defined. Normally, the Python file PC/pyconfig.h does not + cause any of these to be defined, with the exception that _DEBUG + causes Py_DEBUG. Double-check that. */ +# ifdef Py_LIMITED_API +# if defined(Py_DEBUG) +# error "pyconfig.h unexpectedly defines Py_DEBUG but _DEBUG is not set" +# endif +# if defined(Py_TRACE_REFS) +# error "pyconfig.h unexpectedly defines Py_TRACE_REFS" +# endif +# if defined(Py_REF_DEBUG) +# error "pyconfig.h unexpectedly defines Py_REF_DEBUG" +# endif +# endif +# else +# include +# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API +# endif # endif #endif diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -247,7 +247,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.11.2" + "\ncompiled with cffi version: 1.11.3" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -295,8 +295,9 @@ base_module_name = self.module_name.split('.')[-1] if self.ffi._embedding is not None: prnt('#define _CFFI_MODULE_NAME "%s"' % (self.module_name,)) - prnt('#define _CFFI_PYTHON_STARTUP_CODE %s' % - (self._string_literal(self.ffi._embedding),)) + prnt('static const char _CFFI_PYTHON_STARTUP_CODE[] = {') + self._print_string_literal_in_array(self.ffi._embedding) + prnt('0 };') prnt('#ifdef PYPY_VERSION') prnt('# define _CFFI_PYTHON_STARTUP_FUNC _cffi_pypyinit_%s' % ( base_module_name,)) @@ -1271,17 +1272,18 @@ _generate_cpy_extern_python_plus_c_ctx = \ _generate_cpy_extern_python_ctx - def _string_literal(self, s): - def _char_repr(c): - # escape with a '\' the characters '\', '"' or (for trigraphs) '?' - if c in '\\"?': return '\\' + c - if ' ' <= c < '\x7F': return c - if c == '\n': return '\\n' - return '\\%03o' % ord(c) - lines = [] - for line in s.splitlines(True) or ['']: - lines.append('"%s"' % ''.join([_char_repr(c) for c in line])) - return ' \\\n'.join(lines) + def _print_string_literal_in_array(self, s): + prnt = self._prnt + prnt('// # NB. this is not a string because of a size limit in MSVC') + for line in s.splitlines(True): + prnt(('// ' + line).rstrip()) + printed_line = '' + for c in line: + if len(printed_line) >= 76: + prnt(printed_line) + printed_line = '' + printed_line += '%d,' % (ord(c),) + prnt(printed_line) # ---------- # emitting the opcodes for individual types diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py --- a/lib_pypy/cffi/verifier.py +++ b/lib_pypy/cffi/verifier.py @@ -301,7 +301,6 @@ return suffixes def _ensure_dir(filename): - try: - os.makedirs(os.path.dirname(filename)) - except OSError: - pass + dirname = os.path.dirname(filename) + if dirname and not os.path.isdir(dirname): + os.makedirs(dirname) diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi from rpython.rtyper.lltypesystem import rffi -VERSION = "1.11.2" +VERSION = "1.11.3" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.11.2", ("This test_c.py file is for testing a version" +assert __version__ == "1.11.3", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py @@ -2288,3 +2288,13 @@ def test_char16_char32_plain_c(): test_char16_char32_type(no_cpp=True) + +def test_loader_spec(): + ffi = FFI() + lib = verify(ffi, "test_loader_spec", "") + if sys.version_info < (3,): + assert not hasattr(lib, '__loader__') + assert not hasattr(lib, '__spec__') + else: + assert lib.__loader__ is None + assert lib.__spec__ is None diff --git a/rpython/rlib/rwinreg.py b/rpython/rlib/rwinreg.py --- a/rpython/rlib/rwinreg.py +++ b/rpython/rlib/rwinreg.py @@ -47,7 +47,7 @@ HKEY = rwin32.HANDLE PHKEY = rffi.CArrayPtr(HKEY) REGSAM = rwin32.DWORD -suffix = 'W' +suffix = 'A' RegSetValue = external( 'RegSetValue' + suffix, From pypy.commits at gmail.com Fri Jan 12 13:59:35 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 12 Jan 2018 10:59:35 -0800 (PST) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <5a590597.1787df0a.ab8a4.586c@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r93660:0ec5e7c14574 Date: 2018-01-12 17:38 +0000 http://bitbucket.org/pypy/pypy/changeset/0ec5e7c14574/ Log: hg merge default diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -50,3 +50,4 @@ 0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0 0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0 09f9160b643e3f02ccb8c843b2fbb4e5cbf54082 release-pypy3.5-v5.10.0 +3f6eaa010fce78cc7973bdc1dfdb95970f08fed2 release-pypy3.5-v5.10.1 diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.11.2 +Version: 1.11.3 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.11.2" -__version_info__ = (1, 11, 2) +__version__ = "1.11.3" +__version_info__ = (1, 11, 3) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h --- a/lib_pypy/cffi/_cffi_include.h +++ b/lib_pypy/cffi/_cffi_include.h @@ -7,11 +7,38 @@ we can learn about Py_DEBUG from pyconfig.h, but it is unclear if the same works for the other two macros. Py_DEBUG implies them, but not the other way around. + + Issue #350: more mess: on Windows, with _MSC_VER, we have to define + Py_LIMITED_API even before including pyconfig.h. In that case, we + guess what pyconfig.h will do to the macros above, and check our + guess after the #include. */ #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) -# include -# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) -# define Py_LIMITED_API +# ifdef _MSC_VER +# if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API +# endif +# include + /* sanity-check: Py_LIMITED_API will cause crashes if any of these + are also defined. Normally, the Python file PC/pyconfig.h does not + cause any of these to be defined, with the exception that _DEBUG + causes Py_DEBUG. Double-check that. */ +# ifdef Py_LIMITED_API +# if defined(Py_DEBUG) +# error "pyconfig.h unexpectedly defines Py_DEBUG but _DEBUG is not set" +# endif +# if defined(Py_TRACE_REFS) +# error "pyconfig.h unexpectedly defines Py_TRACE_REFS" +# endif +# if defined(Py_REF_DEBUG) +# error "pyconfig.h unexpectedly defines Py_REF_DEBUG" +# endif +# endif +# else +# include +# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API +# endif # endif #endif diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -247,7 +247,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.11.2" + "\ncompiled with cffi version: 1.11.3" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -295,8 +295,9 @@ base_module_name = self.module_name.split('.')[-1] if self.ffi._embedding is not None: prnt('#define _CFFI_MODULE_NAME "%s"' % (self.module_name,)) - prnt('#define _CFFI_PYTHON_STARTUP_CODE %s' % - (self._string_literal(self.ffi._embedding),)) + prnt('static const char _CFFI_PYTHON_STARTUP_CODE[] = {') + self._print_string_literal_in_array(self.ffi._embedding) + prnt('0 };') prnt('#ifdef PYPY_VERSION') prnt('# define _CFFI_PYTHON_STARTUP_FUNC _cffi_pypyinit_%s' % ( base_module_name,)) @@ -1271,17 +1272,18 @@ _generate_cpy_extern_python_plus_c_ctx = \ _generate_cpy_extern_python_ctx - def _string_literal(self, s): - def _char_repr(c): - # escape with a '\' the characters '\', '"' or (for trigraphs) '?' - if c in '\\"?': return '\\' + c - if ' ' <= c < '\x7F': return c - if c == '\n': return '\\n' - return '\\%03o' % ord(c) - lines = [] - for line in s.splitlines(True) or ['']: - lines.append('"%s"' % ''.join([_char_repr(c) for c in line])) - return ' \\\n'.join(lines) + def _print_string_literal_in_array(self, s): + prnt = self._prnt + prnt('// # NB. this is not a string because of a size limit in MSVC') + for line in s.splitlines(True): + prnt(('// ' + line).rstrip()) + printed_line = '' + for c in line: + if len(printed_line) >= 76: + prnt(printed_line) + printed_line = '' + printed_line += '%d,' % (ord(c),) + prnt(printed_line) # ---------- # emitting the opcodes for individual types diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py --- a/lib_pypy/cffi/verifier.py +++ b/lib_pypy/cffi/verifier.py @@ -301,7 +301,6 @@ return suffixes def _ensure_dir(filename): - try: - os.makedirs(os.path.dirname(filename)) - except OSError: - pass + dirname = os.path.dirname(filename) + if dirname and not os.path.isdir(dirname): + os.makedirs(dirname) diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi from rpython.rtyper.lltypesystem import rffi -VERSION = "1.11.2" +VERSION = "1.11.3" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.11.2", ("This test_c.py file is for testing a version" +assert __version__ == "1.11.3", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py @@ -2288,3 +2288,13 @@ def test_char16_char32_plain_c(): test_char16_char32_type(no_cpp=True) + +def test_loader_spec(): + ffi = FFI() + lib = verify(ffi, "test_loader_spec", "") + if sys.version_info < (3,): + assert not hasattr(lib, '__loader__') + assert not hasattr(lib, '__spec__') + else: + assert lib.__loader__ is None + assert lib.__spec__ is None diff --git a/rpython/rlib/rwinreg.py b/rpython/rlib/rwinreg.py --- a/rpython/rlib/rwinreg.py +++ b/rpython/rlib/rwinreg.py @@ -47,7 +47,7 @@ HKEY = rwin32.HANDLE PHKEY = rffi.CArrayPtr(HKEY) REGSAM = rwin32.DWORD -suffix = 'W' +suffix = 'A' RegSetValue = external( 'RegSetValue' + suffix, From pypy.commits at gmail.com Fri Jan 12 13:59:39 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 12 Jan 2018 10:59:39 -0800 (PST) Subject: [pypy-commit] pypy py3.5-refactor-slots: hg merge refactor-slots Message-ID: <5a59059b.493f1c0a.d713e.ca90@mx.google.com> Author: Ronan Lamy Branch: py3.5-refactor-slots Changeset: r93662:de570a6daa50 Date: 2018-01-12 18:58 +0000 http://bitbucket.org/pypy/pypy/changeset/de570a6daa50/ Log: hg merge refactor-slots diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -420,229 +420,7 @@ def build_slot_tp_function(space, typedef, name, method_name): - w_type = space.gettypeobject(typedef) - - handled = False - # unary functions - for tp_name, attr in [('tp_hash', '__hash__'), - ('tp_as_sequence.c_sq_length', '__len__'), - ('tp_as_mapping.c_mp_length', '__len__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - @slot_function([PyObject], lltype.Signed, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_obj): - return space.int_w(space.call_function(slot_fn, w_obj)) - handled = True - - - # binary functions - for tp_name, attr in [('tp_as_number.c_nb_add', '__add__'), - ('tp_as_number.c_nb_subtract', '__sub__'), - ('tp_as_number.c_nb_multiply', '__mul__'), - ('tp_as_number.c_nb_divide', '__div__'), - ('tp_as_number.c_nb_remainder', '__mod__'), - ('tp_as_number.c_nb_divmod', '__divmod__'), - ('tp_as_number.c_nb_lshift', '__lshift__'), - ('tp_as_number.c_nb_rshift', '__rshift__'), - ('tp_as_number.c_nb_and', '__and__'), - ('tp_as_number.c_nb_xor', '__xor__'), - ('tp_as_number.c_nb_or', '__or__'), - ('tp_as_sequence.c_sq_concat', '__add__'), - ('tp_as_sequence.c_sq_inplace_concat', '__iadd__'), - ('tp_as_mapping.c_mp_subscript', '__getitem__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - - @slot_function([PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, w_arg): - return space.call_function(slot_fn, w_self, w_arg) - handled = True - - # binary-with-Py_ssize_t-type - for tp_name, attr in [('tp_as_sequence.c_sq_item', '__getitem__'), - ('tp_as_sequence.c_sq_repeat', '__mul__'), - ('tp_as_sequence.c_sq_repeat', '__mul__'), - ('tp_as_sequence.c_sq_inplace_repeat', '__imul__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - - @slot_function([PyObject, Py_ssize_t], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, arg): - return space.call_function(slot_fn, w_self, space.newint(arg)) - handled = True - - # ternary functions - for tp_name, attr in [('tp_as_number.c_nb_power', '__pow__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, w_arg1, w_arg2): - return space.call_function(slot_fn, w_self, w_arg1, w_arg2) - handled = True - # ternary-with-void returning-Py_size_t-type - for tp_name, attr in [('tp_as_mapping.c_mp_ass_subscript', '__setitem__'), - ]: - if name == tp_name: - slot_ass = w_type.lookup(attr) - if slot_ass is None: - return - slot_del = w_type.lookup('__delitem__') - if slot_del is None: - return - - @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, w_arg1, arg2): - if arg2: - w_arg2 = from_ref(space, rffi.cast(PyObject, arg2)) - space.call_function(slot_ass, w_self, w_arg1, w_arg2) - else: - space.call_function(slot_del, w_self, w_arg1) - return 0 - handled = True - # ternary-Py_size_t-void returning-Py_size_t-type - for tp_name, attr in [('tp_as_sequence.c_sq_ass_item', '__setitem__'), - ]: - if name == tp_name: - slot_ass = w_type.lookup(attr) - if slot_ass is None: - return - slot_del = w_type.lookup('__delitem__') - if slot_del is None: - return - - @slot_function([PyObject, lltype.Signed, PyObject], rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, arg1, arg2): - if arg2: - w_arg2 = from_ref(space, rffi.cast(PyObject, arg2)) - space.call_function(slot_ass, w_self, space.newint(arg1), w_arg2) - else: - space.call_function(slot_del, w_self, space.newint(arg1)) - return 0 - handled = True - if handled: - pass - elif name == 'tp_setattro': - setattr_fn = w_type.lookup('__setattr__') - delattr_fn = w_type.lookup('__delattr__') - if setattr_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, - error=-1) - @func_renamer("cpyext_tp_setattro_%s" % (typedef.name,)) - def slot_tp_setattro(space, w_self, w_name, w_value): - if w_value is not None: - space.call_function(setattr_fn, w_self, w_name, w_value) - else: - space.call_function(delattr_fn, w_self, w_name) - return 0 - slot_func = slot_tp_setattro - elif name == 'tp_getattro': - getattr_fn = w_type.lookup('__getattribute__') - if getattr_fn is None: - return - - @slot_function([PyObject, PyObject], PyObject) - @func_renamer("cpyext_tp_getattro_%s" % (typedef.name,)) - def slot_tp_getattro(space, w_self, w_name): - return space.call_function(getattr_fn, w_self, w_name) - slot_func = slot_tp_getattro - - elif name == 'tp_call': - call_fn = w_type.lookup('__call__') - if call_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_call(space, w_self, w_args, w_kwds): - args = Arguments(space, [w_self], - w_stararg=w_args, w_starstararg=w_kwds) - return space.call_args(call_fn, args) - slot_func = slot_tp_call - - elif name == 'tp_iternext': - iternext_fn = w_type.lookup('__next__') - if iternext_fn is None: - return - - @slot_function([PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_iternext(space, w_self): - try: - return space.call_function(iternext_fn, w_self) - except OperationError as e: - if not e.match(space, space.w_StopIteration): - raise - return None - slot_func = slot_tp_iternext - - elif name == 'tp_init': - init_fn = w_type.lookup('__init__') - if init_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_init(space, w_self, w_args, w_kwds): - args = Arguments(space, [w_self], - w_stararg=w_args, w_starstararg=w_kwds) - space.call_args(init_fn, args) - return 0 - slot_func = slot_tp_init - elif name == 'tp_new': - new_fn = w_type.lookup('__new__') - if new_fn is None: - return - - @slot_function([PyTypeObjectPtr, PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_new(space, w_self, w_args, w_kwds): - args = Arguments(space, [w_self], - w_stararg=w_args, w_starstararg=w_kwds) - return space.call_args(space.get(new_fn, w_self), args) - slot_func = slot_tp_new - elif name == 'tp_as_buffer.c_bf_getbuffer': - buff_fn = w_type.lookup('__buffer__') - if buff_fn is not None: - buff_w = slot_from___buffer__(space, typedef, buff_fn) - elif typedef.buffer: - buff_w = slot_from_buffer_w(space, typedef, buff_fn) - else: - return - slot_func = buff_w - elif name == 'tp_descr_get': - get_fn = w_type.lookup('__get__') - if get_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_descr_get(space, w_self, w_obj, w_value): - if w_obj is None: - w_obj = space.w_None - return space.call_function(get_fn, w_self, w_obj, w_value) - slot_func = slot_tp_descr_get - elif name in SLOT_FACTORIES: + if name in SLOT_FACTORIES: return SLOT_FACTORIES[name](space, typedef, name, method_name) else: # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce @@ -650,7 +428,6 @@ # richcmpfunc(s) return - return slot_func def make_unary_slot(space, typedef, name, attr): w_type = space.gettypeobject(typedef) @@ -664,7 +441,6 @@ return space.call_function(slot_fn, w_self) return slot_func - UNARY_SLOTS = [ 'tp_as_async.c_am_await', 'tp_as_async.c_am_anext', @@ -685,10 +461,256 @@ for name in UNARY_SLOTS: slot_factory(name)(make_unary_slot) +def make_unary_slot_int(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_fn = w_type.lookup(attr) + if slot_fn is None: + return + @slot_function([PyObject], lltype.Signed, error=-1) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_obj): + return space.int_w(space.call_function(slot_fn, w_obj)) + return slot_func + +UNARY_SLOTS_INT = [ + 'tp_hash', + 'tp_as_sequence.c_sq_length', + 'tp_as_mapping.c_mp_length',] +for name in UNARY_SLOTS_INT: + slot_factory(name)(make_unary_slot_int) + + +def make_binary_slot(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_fn = w_type.lookup(attr) + if slot_fn is None: + return + + @slot_function([PyObject, PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self, w_arg): + return space.call_function(slot_fn, w_self, w_arg) + return slot_func + +BINARY_SLOTS = [ + 'tp_as_number.c_nb_add', + 'tp_as_number.c_nb_subtract', + 'tp_as_number.c_nb_multiply', + 'tp_as_number.c_nb_divide', + 'tp_as_number.c_nb_remainder', + 'tp_as_number.c_nb_divmod', + 'tp_as_number.c_nb_lshift', + 'tp_as_number.c_nb_rshift', + 'tp_as_number.c_nb_and', + 'tp_as_number.c_nb_xor', + 'tp_as_number.c_nb_or', + 'tp_as_sequence.c_sq_concat', + 'tp_as_sequence.c_sq_inplace_concat', + 'tp_as_mapping.c_mp_subscript',] +for name in BINARY_SLOTS: + slot_factory(name)(make_binary_slot) + + +def make_binary_slot_int(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_fn = w_type.lookup(attr) + if slot_fn is None: + return + + @slot_function([PyObject, Py_ssize_t], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self, arg): + return space.call_function(slot_fn, w_self, space.newint(arg)) + return slot_func + +BINARY_SLOTS_INT = [ + 'tp_as_sequence.c_sq_item', + 'tp_as_sequence.c_sq_repeat', + 'tp_as_sequence.c_sq_repeat', + 'tp_as_sequence.c_sq_inplace_repeat',] +for name in BINARY_SLOTS_INT: + slot_factory(name)(make_binary_slot_int) + + at slot_factory('tp_as_number.c_nb_power') +def make_nb_power(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_fn = w_type.lookup(attr) + if slot_fn is None: + return + + @slot_function([PyObject, PyObject, PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self, w_arg1, w_arg2): + return space.call_function(slot_fn, w_self, w_arg1, w_arg2) + return slot_func + + at slot_factory('tp_as_mapping.c_mp_ass_subscript') +def make_sq_set_item(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_ass = w_type.lookup(attr) + if slot_ass is None: + return + slot_del = w_type.lookup('__delitem__') + if slot_del is None: + return + + @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self, w_arg1, arg2): + if arg2: + w_arg2 = from_ref(space, rffi.cast(PyObject, arg2)) + space.call_function(slot_ass, w_self, w_arg1, w_arg2) + else: + space.call_function(slot_del, w_self, w_arg1) + return 0 + return slot_func + + + at slot_factory('tp_as_sequence.c_sq_ass_item') +def make_sq_ass_item(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_ass = w_type.lookup(attr) + if slot_ass is None: + return + slot_del = w_type.lookup('__delitem__') + if slot_del is None: + return + + @slot_function([PyObject, lltype.Signed, PyObject], rffi.INT_real, error=-1) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self, arg1, arg2): + if arg2: + w_arg2 = from_ref(space, rffi.cast(PyObject, arg2)) + space.call_function(slot_ass, w_self, space.newint(arg1), w_arg2) + else: + space.call_function(slot_del, w_self, space.newint(arg1)) + return 0 + return slot_func + +def make_tp_setattro(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + setattr_fn = w_type.lookup('__setattr__') + delattr_fn = w_type.lookup('__delattr__') + if setattr_fn is None or delattr_fn is None: + return + + @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, + error=-1) + @func_renamer("cpyext_tp_setattro_%s" % (typedef.name,)) + def slot_tp_setattro(space, w_self, w_name, w_value): + if w_value is not None: + space.call_function(setattr_fn, w_self, w_name, w_value) + else: + space.call_function(delattr_fn, w_self, w_name) + return 0 + return slot_tp_setattro + + at slot_factory('tp_getattro') +def make_tp_getattro(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + getattr_fn = w_type.lookup('__getattribute__') + if getattr_fn is None: + return + + @slot_function([PyObject, PyObject], PyObject) + @func_renamer("cpyext_tp_getattro_%s" % (typedef.name,)) + def slot_tp_getattro(space, w_self, w_name): + return space.call_function(getattr_fn, w_self, w_name) + return slot_tp_getattro + + at slot_factory('tp_call') +def make_tp_call(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + call_fn = w_type.lookup('__call__') + if call_fn is None: + return + + @slot_function([PyObject, PyObject, PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_tp_call(space, w_self, w_args, w_kwds): + args = Arguments(space, [w_self], + w_stararg=w_args, w_starstararg=w_kwds) + return space.call_args(call_fn, args) + return slot_tp_call + + at slot_factory('tp_iternext') +def make_tp_iternext(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + iternext_fn = w_type.lookup('__next__') + if iternext_fn is None: + return + + @slot_function([PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_tp_iternext(space, w_self): + try: + return space.call_function(iternext_fn, w_self) + except OperationError as e: + if not e.match(space, space.w_StopIteration): + raise + return None + return slot_tp_iternext + + at slot_factory('tp_init') +def make_tp_init(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + init_fn = w_type.lookup('__init__') + if init_fn is None: + return + + @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_tp_init(space, w_self, w_args, w_kwds): + args = Arguments(space, [w_self], + w_stararg=w_args, w_starstararg=w_kwds) + space.call_args(init_fn, args) + return 0 + return slot_tp_init + + at slot_factory('tp_new') +def make_tp_new(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + new_fn = w_type.lookup('__new__') + if new_fn is None: + return + + @slot_function([PyTypeObjectPtr, PyObject, PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_tp_new(space, w_self, w_args, w_kwds): + args = Arguments(space, [w_self], + w_stararg=w_args, w_starstararg=w_kwds) + return space.call_args(space.get(new_fn, w_self), args) + return slot_tp_new + + at slot_factory('tp_as_buffer.c_bf_getbuffer') +def make_bf_getbuffer(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + buff_fn = w_type.lookup('__buffer__') + if buff_fn is not None: + return slot_from___buffer__(space, typedef, buff_fn) + elif typedef.buffer: + return slot_from_buffer_w(space, typedef) + else: + return None + + at slot_factory('tp_descr_get') +def make_tp_descr_get(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + get_fn = w_type.lookup('__get__') + if get_fn is None: + return + + @slot_function([PyObject, PyObject, PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_tp_descr_get(space, w_self, w_obj, w_value): + if w_obj is None: + w_obj = space.w_None + return space.call_function(get_fn, w_self, w_obj, w_value) + return slot_tp_descr_get + @slot_factory('tp_descr_set') def make_tp_descr_set(space, typedef, name, attr): w_type = space.gettypeobject(typedef) - name = 'descr_set' set_fn = w_type.lookup('__set__') delete_fn = w_type.lookup('__delete__') if set_fn is None and delete_fn is None: @@ -738,7 +760,7 @@ return 0 return buff_w -def slot_from_buffer_w(space, typedef, buff_fn): +def slot_from_buffer_w(space, typedef): name = 'bf_getbuffer' @slot_function([PyObject, Py_bufferP, rffi.INT_real], rffi.INT_real, error=-1) From pypy.commits at gmail.com Sat Jan 13 04:47:07 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 13 Jan 2018 01:47:07 -0800 (PST) Subject: [pypy-commit] cffi doc-index: Close branch doc-index Message-ID: <5a59d59b.0a9fdf0a.9c2ea.3dc5@mx.google.com> Author: Armin Rigo Branch: doc-index Changeset: r3077:51a0179a86d8 Date: 2018-01-13 09:47 +0000 http://bitbucket.org/cffi/cffi/changeset/51a0179a86d8/ Log: Close branch doc-index From pypy.commits at gmail.com Sat Jan 13 04:47:10 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 13 Jan 2018 01:47:10 -0800 (PST) Subject: [pypy-commit] cffi default: Merged in doc-index (pull request #85) Message-ID: <5a59d59e.8be41c0a.a486a.d77f@mx.google.com> Author: Armin Rigo Branch: Changeset: r3078:eccbb9868f8a Date: 2018-01-13 09:47 +0000 http://bitbucket.org/cffi/cffi/changeset/eccbb9868f8a/ Log: Merged in doc-index (pull request #85) split index and lower headings of old versions so TOC looks nicer diff --git a/doc/source/goals.rst b/doc/source/goals.rst new file mode 100644 --- /dev/null +++ b/doc/source/goals.rst @@ -0,0 +1,62 @@ +Goals +----- + +The interface is based on `LuaJIT's FFI`_, and follows a few principles: + +* The goal is to call C code from Python without learning a 3rd language: + existing alternatives require users to learn domain specific language + (Cython_, SWIG_) or API (ctypes_). The CFFI design requires users to know + only C and Python, minimizing the extra bits of API that need to be learned. + +* Keep all the Python-related logic in Python so that you don't need to + write much C code (unlike `CPython native C extensions`_). + +* The preferred way is to work at the level of the API (Application + Programming Interface): the C compiler is called from the declarations + you write to validate and link to the C language constructs. + Alternatively, it is also possible to work at the ABI level + (Application Binary Interface), the way ctypes_ work. + However, on non-Windows platforms, C libraries typically + have a specified C API but not an ABI (e.g. they may + document a "struct" as having at least these fields, but maybe more). + +* Try to be complete. For now some C99 constructs are not supported, + but all C89 should be, including macros (and including macro "abuses", + which you can `manually wrap`_ in saner-looking C functions). + +* Attempt to support both PyPy and CPython, with a reasonable path + for other Python implementations like IronPython and Jython. + +* Note that this project is **not** about embedding executable C code in + Python, unlike `Weave`_. This is about calling existing C libraries + from Python. + +.. _`LuaJIT's FFI`: http://luajit.org/ext_ffi.html +.. _`Cython`: http://www.cython.org +.. _`SWIG`: http://www.swig.org/ +.. _`CPython native C extensions`: http://docs.python.org/extending/extending.html +.. _`native C extensions`: http://docs.python.org/extending/extending.html +.. _`ctypes`: http://docs.python.org/library/ctypes.html +.. _`Weave`: http://wiki.scipy.org/Weave +.. _`manually wrap`: overview.html#abi-versus-api + +Get started by reading `the overview`__. + +.. __: overview.html + + +Comments and bugs +----------------- + +The best way to contact us is on the IRC ``#pypy`` channel of +``irc.freenode.net``. Feel free to discuss matters either there or in +the `mailing list`_. Please report to the `issue tracker`_ any bugs. + +As a general rule, when there is a design issue to resolve, we pick the +solution that is the "most C-like". We hope that this module has got +everything you need to access C code and nothing more. + +--- the authors, Armin Rigo and Maciej Fijalkowski + +.. _`issue tracker`: https://bitbucket.org/cffi/cffi/issues +.. _`mailing list`: https://groups.google.com/forum/#!forum/python-cffi diff --git a/doc/source/index.rst b/doc/source/index.rst --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -6,13 +6,10 @@ code from Python, based on C-like declarations that you can often copy-paste from header files or documentation. -* Goals_ - - * `Comments and bugs`_ - .. toctree:: :maxdepth: 2 + goals whatsnew installation overview @@ -22,65 +19,4 @@ embedding -Goals ------ -The interface is based on `LuaJIT's FFI`_, and follows a few principles: - -* The goal is to call C code from Python without learning a 3rd language: - existing alternatives require users to learn domain specific language - (Cython_, SWIG_) or API (ctypes_). The CFFI design requires users to know - only C and Python, minimizing the extra bits of API that need to be learned. - -* Keep all the Python-related logic in Python so that you don't need to - write much C code (unlike `CPython native C extensions`_). - -* The preferred way is to work at the level of the API (Application - Programming Interface): the C compiler is called from the declarations - you write to validate and link to the C language constructs. - Alternatively, it is also possible to work at the ABI level - (Application Binary Interface), the way ctypes_ work. - However, on non-Windows platforms, C libraries typically - have a specified C API but not an ABI (e.g. they may - document a "struct" as having at least these fields, but maybe more). - -* Try to be complete. For now some C99 constructs are not supported, - but all C89 should be, including macros (and including macro "abuses", - which you can `manually wrap`_ in saner-looking C functions). - -* Attempt to support both PyPy and CPython, with a reasonable path - for other Python implementations like IronPython and Jython. - -* Note that this project is **not** about embedding executable C code in - Python, unlike `Weave`_. This is about calling existing C libraries - from Python. - -.. _`LuaJIT's FFI`: http://luajit.org/ext_ffi.html -.. _`Cython`: http://www.cython.org -.. _`SWIG`: http://www.swig.org/ -.. _`CPython native C extensions`: http://docs.python.org/extending/extending.html -.. _`native C extensions`: http://docs.python.org/extending/extending.html -.. _`ctypes`: http://docs.python.org/library/ctypes.html -.. _`Weave`: http://wiki.scipy.org/Weave -.. _`manually wrap`: overview.html#abi-versus-api - -Get started by reading `the overview`__. - -.. __: overview.html - - -Comments and bugs ------------------ - -The best way to contact us is on the IRC ``#pypy`` channel of -``irc.freenode.net``. Feel free to discuss matters either there or in -the `mailing list`_. Please report to the `issue tracker`_ any bugs. - -As a general rule, when there is a design issue to resolve, we pick the -solution that is the "most C-like". We hope that this module has got -everything you need to access C code and nothing more. - ---- the authors, Armin Rigo and Maciej Fijalkowski - -.. _`issue tracker`: https://bitbucket.org/cffi/cffi/issues -.. _`mailing list`: https://groups.google.com/forum/#!forum/python-cffi diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst --- a/doc/source/whatsnew.rst +++ b/doc/source/whatsnew.rst @@ -95,8 +95,11 @@ .. __: http://bugs.python.org/issue31105 +Older Versions +============== + v1.10.1 -======= +------- (only released inside PyPy 5.8.0) @@ -107,7 +110,7 @@ v1.10 -===== +----- * Issue #295: use calloc() directly instead of PyObject_Malloc()+memset() to handle ffi.new() with a default @@ -169,7 +172,7 @@ v1.9 -==== +---- * Structs with variable-sized arrays as their last field: now we track the length of the array after ``ffi.new()`` is called, just like we @@ -204,7 +207,7 @@ v1.8.3 -====== +------ * When passing a ``void *`` argument to a function with a different pointer type, or vice-versa, the cast occurs automatically, like in C. @@ -217,7 +220,7 @@ v1.8.2 -====== +------ * Issue #283: fixed ``ffi.new()`` on structures/unions with nested anonymous structures/unions, when there is at least one union in @@ -226,7 +229,7 @@ v1.8.1 -====== +------ * CPython 3.x: experimental: the generated C extension modules now use the "limited API", which means that, as a compiled .so/.dll, it should @@ -241,7 +244,7 @@ v1.8 -==== +---- * Removed the restriction that ``ffi.from_buffer()`` cannot be used on byte strings. Now you can get a ``char *`` out of a byte string, @@ -255,7 +258,7 @@ v1.7 -==== +---- * ``ffi.gc(p, None)`` removes the destructor on an object previously created by another call to ``ffi.gc()`` @@ -284,7 +287,7 @@ v1.6 -==== +---- * `ffi.list_types()`_ @@ -307,13 +310,13 @@ v1.5.2 -====== +------ * Fix 1.5.1 for Python 2.6. v1.5.1 -====== +------ * A few installation-time tweaks (thanks Stefano!) @@ -325,7 +328,7 @@ v1.5.0 -====== +------ * Support for `using CFFI for embedding`__. @@ -333,13 +336,13 @@ v1.4.2 -====== +------ Nothing changed from v1.4.1. v1.4.1 -====== +------ * Fix the compilation failure of cffi on CPython 3.5.0. (3.5.1 works; some detail changed that makes some underscore-starting macros @@ -349,7 +352,7 @@ v1.4.0 -====== +------ * A `better way to do callbacks`__ has been added (faster and more portable, and usually cleaner). It is a mechanism for the @@ -390,7 +393,7 @@ v1.3.1 -====== +------ * The optional typedefs (``bool``, ``FILE`` and all Windows types) were not always available from out-of-line FFI objects. @@ -406,7 +409,7 @@ v1.3.0 -====== +------ * Added `ffi.memmove()`_. @@ -441,13 +444,13 @@ v1.2.1 -====== +------ Nothing changed from v1.2.0. v1.2.0 -====== +------ * Out-of-line mode: ``int a[][...];`` can be used to declare a structure field or global variable which is, simultaneously, of total length @@ -500,14 +503,14 @@ v1.1.2 -====== +------ * ``ffi.gc()``: fixed a race condition in multithreaded programs introduced in 1.1.1 v1.1.1 -====== +------ * Out-of-line mode: ``ffi.string()``, ``ffi.buffer()`` and ``ffi.getwinerror()`` didn't accept their arguments as keyword @@ -525,7 +528,7 @@ v1.1.0 -====== +------ * Out-of-line API mode: we can now declare integer types with ``typedef int... foo_t;``. The exact size and signedness of ``foo_t`` @@ -558,13 +561,13 @@ v1.0.3 -====== +------ * Same as 1.0.2, apart from doc and test fixes on some platforms. v1.0.2 -====== +------ * Variadic C functions (ending in a "..." argument) were not supported in the out-of-line ABI mode. This was a bug---there was even a @@ -574,7 +577,7 @@ v1.0.1 -====== +------ * ``ffi.set_source()`` crashed if passed a ``sources=[..]`` argument. Fixed by chrippa on pull request #60. @@ -587,7 +590,7 @@ v1.0.0 -====== +------ * The main news item is out-of-line module generation: From pypy.commits at gmail.com Sat Jan 13 04:56:00 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 13 Jan 2018 01:56:00 -0800 (PST) Subject: [pypy-commit] cffi default: Issue #355 Message-ID: <5a59d7b0.23addf0a.9a3d1.0216@mx.google.com> Author: Armin Rigo Branch: Changeset: r3079:4b3fb90d4418 Date: 2018-01-13 10:55 +0100 http://bitbucket.org/cffi/cffi/changeset/4b3fb90d4418/ Log: Issue #355 Revert 164e526a5515 and 14ce6985e1c3 and add an explanation about why. diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h --- a/cffi/_cffi_include.h +++ b/cffi/_cffi_include.h @@ -8,37 +8,19 @@ the same works for the other two macros. Py_DEBUG implies them, but not the other way around. - Issue #350: more mess: on Windows, with _MSC_VER, we have to define - Py_LIMITED_API even before including pyconfig.h. In that case, we - guess what pyconfig.h will do to the macros above, and check our - guess after the #include. + Issue #350 is still open: on Windows, the code here causes it to link + with PYTHON36.DLL (for example) instead of PYTHON3.DLL. A fix was + attempted in 164e526a5515 and 14ce6985e1c3, but reverted: virtualenv + does not make PYTHON3.DLL available, and so the "correctly" compiled + version would not run inside a virtualenv. We will re-apply the fix + after virtualenv has been fixed for some time. For explanation, see + issue #355. For a workaround if you want PYTHON3.DLL and don't worry + about virtualenv, see issue #350. */ #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) -# ifdef _MSC_VER -# if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) -# define Py_LIMITED_API -# endif -# include - /* sanity-check: Py_LIMITED_API will cause crashes if any of these - are also defined. Normally, the Python file PC/pyconfig.h does not - cause any of these to be defined, with the exception that _DEBUG - causes Py_DEBUG. Double-check that. */ -# ifdef Py_LIMITED_API -# if defined(Py_DEBUG) -# error "pyconfig.h unexpectedly defines Py_DEBUG but _DEBUG is not set" -# endif -# if defined(Py_TRACE_REFS) -# error "pyconfig.h unexpectedly defines Py_TRACE_REFS" -# endif -# if defined(Py_REF_DEBUG) -# error "pyconfig.h unexpectedly defines Py_REF_DEBUG" -# endif -# endif -# else -# include -# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) -# define Py_LIMITED_API -# endif +# include +# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API # endif #endif From pypy.commits at gmail.com Sat Jan 13 05:37:30 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 13 Jan 2018 02:37:30 -0800 (PST) Subject: [pypy-commit] cffi default: Update whatsnew Message-ID: <5a59e16a.c2b2df0a.6278.e8bc@mx.google.com> Author: Armin Rigo Branch: Changeset: r3080:aaf620bfeb97 Date: 2018-01-13 10:58 +0100 http://bitbucket.org/cffi/cffi/changeset/aaf620bfeb97/ Log: Update whatsnew diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst --- a/doc/source/whatsnew.rst +++ b/doc/source/whatsnew.rst @@ -2,6 +2,15 @@ What's New ====================== +v1.11.4 +======= + +* Windows: reverted linking with ``python3.dll``, because + virtualenv does not make this DLL available to virtual environments + for now. See `Issue #355`_. + +.. _`Issue #355`: https://bitbucket.org/cffi/cffi/issues/355/ + v1.11.3 ======= From pypy.commits at gmail.com Sat Jan 13 05:37:32 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 13 Jan 2018 02:37:32 -0800 (PST) Subject: [pypy-commit] cffi default: Bump version number to 1.11.4 Message-ID: <5a59e16c.14121c0a.fcf65.7c86@mx.google.com> Author: Armin Rigo Branch: Changeset: r3081:237ed9a1a804 Date: 2018-01-13 11:37 +0100 http://bitbucket.org/cffi/cffi/changeset/237ed9a1a804/ Log: Bump version number to 1.11.4 diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -2,7 +2,7 @@ #include #include "structmember.h" -#define CFFI_VERSION "1.11.3" +#define CFFI_VERSION "1.11.4" #ifdef MS_WIN32 #include diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -12,7 +12,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.11.3", ("This test_c.py file is for testing a version" +assert __version__ == "1.11.4", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/cffi/__init__.py b/cffi/__init__.py --- a/cffi/__init__.py +++ b/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.11.3" -__version_info__ = (1, 11, 3) +__version__ = "1.11.4" +__version_info__ = (1, 11, 4) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/cffi/_embedding.h b/cffi/_embedding.h --- a/cffi/_embedding.h +++ b/cffi/_embedding.h @@ -247,7 +247,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.11.3" + "\ncompiled with cffi version: 1.11.4" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/doc/source/conf.py b/doc/source/conf.py --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -47,7 +47,7 @@ # The short X.Y version. version = '1.11' # The full version, including alpha/beta/rc tags. -release = '1.11.3' +release = '1.11.4' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/installation.rst b/doc/source/installation.rst --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -53,13 +53,13 @@ * https://pypi.python.org/pypi/cffi -* Checksums of the "source" package version 1.11.3: +* Checksums of the "source" package version 1.11.4: - - MD5: 3d263ec84bb9ad08e0edf2d21312fa1e + - MD5: ... - - SHA: 26a73cc075064cc7cd59ec6c58ca0684d16c4ece + - SHA: ... - - SHA256: 150708ce9e417858f9a4056b5f364d8c7077fd979b9e35307c2d8a4a8e991fd2 + - SHA256: ... * Or grab the most current version from the `Bitbucket page`_: ``hg clone https://bitbucket.org/cffi/cffi`` diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -186,7 +186,7 @@ `Mailing list `_ """, - version='1.11.3', + version='1.11.4', packages=['cffi'] if cpython else [], package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', '_embedding.h', '_cffi_errors.h']} From pypy.commits at gmail.com Sat Jan 13 05:58:16 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 13 Jan 2018 02:58:16 -0800 (PST) Subject: [pypy-commit] cffi release-1.11: hg merge default Message-ID: <5a59e648.c9061c0a.6c603.2d56@mx.google.com> Author: Armin Rigo Branch: release-1.11 Changeset: r3082:fa470f261b0d Date: 2018-01-13 11:57 +0100 http://bitbucket.org/cffi/cffi/changeset/fa470f261b0d/ Log: hg merge default diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -2,7 +2,7 @@ #include #include "structmember.h" -#define CFFI_VERSION "1.11.3" +#define CFFI_VERSION "1.11.4" #ifdef MS_WIN32 #include diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -12,7 +12,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.11.3", ("This test_c.py file is for testing a version" +assert __version__ == "1.11.4", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/cffi/__init__.py b/cffi/__init__.py --- a/cffi/__init__.py +++ b/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.11.3" -__version_info__ = (1, 11, 3) +__version__ = "1.11.4" +__version_info__ = (1, 11, 4) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h --- a/cffi/_cffi_include.h +++ b/cffi/_cffi_include.h @@ -8,37 +8,19 @@ the same works for the other two macros. Py_DEBUG implies them, but not the other way around. - Issue #350: more mess: on Windows, with _MSC_VER, we have to define - Py_LIMITED_API even before including pyconfig.h. In that case, we - guess what pyconfig.h will do to the macros above, and check our - guess after the #include. + Issue #350 is still open: on Windows, the code here causes it to link + with PYTHON36.DLL (for example) instead of PYTHON3.DLL. A fix was + attempted in 164e526a5515 and 14ce6985e1c3, but reverted: virtualenv + does not make PYTHON3.DLL available, and so the "correctly" compiled + version would not run inside a virtualenv. We will re-apply the fix + after virtualenv has been fixed for some time. For explanation, see + issue #355. For a workaround if you want PYTHON3.DLL and don't worry + about virtualenv, see issue #350. */ #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) -# ifdef _MSC_VER -# if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) -# define Py_LIMITED_API -# endif -# include - /* sanity-check: Py_LIMITED_API will cause crashes if any of these - are also defined. Normally, the Python file PC/pyconfig.h does not - cause any of these to be defined, with the exception that _DEBUG - causes Py_DEBUG. Double-check that. */ -# ifdef Py_LIMITED_API -# if defined(Py_DEBUG) -# error "pyconfig.h unexpectedly defines Py_DEBUG but _DEBUG is not set" -# endif -# if defined(Py_TRACE_REFS) -# error "pyconfig.h unexpectedly defines Py_TRACE_REFS" -# endif -# if defined(Py_REF_DEBUG) -# error "pyconfig.h unexpectedly defines Py_REF_DEBUG" -# endif -# endif -# else -# include -# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) -# define Py_LIMITED_API -# endif +# include +# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API # endif #endif diff --git a/cffi/_embedding.h b/cffi/_embedding.h --- a/cffi/_embedding.h +++ b/cffi/_embedding.h @@ -247,7 +247,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.11.3" + "\ncompiled with cffi version: 1.11.4" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/doc/source/conf.py b/doc/source/conf.py --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -38,7 +38,7 @@ # General information about the project. project = u'CFFI' -copyright = u'2012-2015, Armin Rigo, Maciej Fijalkowski' +copyright = u'2012-2018, Armin Rigo, Maciej Fijalkowski' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -47,7 +47,7 @@ # The short X.Y version. version = '1.11' # The full version, including alpha/beta/rc tags. -release = '1.11.3' +release = '1.11.4' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/embedding.rst b/doc/source/embedding.rst --- a/doc/source/embedding.rst +++ b/doc/source/embedding.rst @@ -279,12 +279,24 @@ Troubleshooting --------------- -The error message +* The error message cffi extension module 'c_module_name' has unknown version 0x2701 -means that the running Python interpreter located a CFFI version older -than 1.5. CFFI 1.5 or newer must be installed in the running Python. + means that the running Python interpreter located a CFFI version older + than 1.5. CFFI 1.5 or newer must be installed in the running Python. + +* On PyPy, the error message + + debug: pypy_setup_home: directories 'lib-python' and 'lib_pypy' not + found in pypy's shared library location or in any parent directory + + means that the ``libpypy-c.so`` file was found, but the standard library + was not found from this location. This occurs at least on some Linux + distributions, because they put ``libpypy-c.so`` inside ``/usr/lib/``, + instead of the way we recommend, which is: keep that file inside + ``/opt/pypy/bin/`` and put a symlink to there from ``/usr/lib/``. + The quickest fix is to do that change manually. Issues about using the .so diff --git a/doc/source/goals.rst b/doc/source/goals.rst new file mode 100644 --- /dev/null +++ b/doc/source/goals.rst @@ -0,0 +1,62 @@ +Goals +----- + +The interface is based on `LuaJIT's FFI`_, and follows a few principles: + +* The goal is to call C code from Python without learning a 3rd language: + existing alternatives require users to learn domain specific language + (Cython_, SWIG_) or API (ctypes_). The CFFI design requires users to know + only C and Python, minimizing the extra bits of API that need to be learned. + +* Keep all the Python-related logic in Python so that you don't need to + write much C code (unlike `CPython native C extensions`_). + +* The preferred way is to work at the level of the API (Application + Programming Interface): the C compiler is called from the declarations + you write to validate and link to the C language constructs. + Alternatively, it is also possible to work at the ABI level + (Application Binary Interface), the way ctypes_ work. + However, on non-Windows platforms, C libraries typically + have a specified C API but not an ABI (e.g. they may + document a "struct" as having at least these fields, but maybe more). + +* Try to be complete. For now some C99 constructs are not supported, + but all C89 should be, including macros (and including macro "abuses", + which you can `manually wrap`_ in saner-looking C functions). + +* Attempt to support both PyPy and CPython, with a reasonable path + for other Python implementations like IronPython and Jython. + +* Note that this project is **not** about embedding executable C code in + Python, unlike `Weave`_. This is about calling existing C libraries + from Python. + +.. _`LuaJIT's FFI`: http://luajit.org/ext_ffi.html +.. _`Cython`: http://www.cython.org +.. _`SWIG`: http://www.swig.org/ +.. _`CPython native C extensions`: http://docs.python.org/extending/extending.html +.. _`native C extensions`: http://docs.python.org/extending/extending.html +.. _`ctypes`: http://docs.python.org/library/ctypes.html +.. _`Weave`: http://wiki.scipy.org/Weave +.. _`manually wrap`: overview.html#abi-versus-api + +Get started by reading `the overview`__. + +.. __: overview.html + + +Comments and bugs +----------------- + +The best way to contact us is on the IRC ``#pypy`` channel of +``irc.freenode.net``. Feel free to discuss matters either there or in +the `mailing list`_. Please report to the `issue tracker`_ any bugs. + +As a general rule, when there is a design issue to resolve, we pick the +solution that is the "most C-like". We hope that this module has got +everything you need to access C code and nothing more. + +--- the authors, Armin Rigo and Maciej Fijalkowski + +.. _`issue tracker`: https://bitbucket.org/cffi/cffi/issues +.. _`mailing list`: https://groups.google.com/forum/#!forum/python-cffi diff --git a/doc/source/index.rst b/doc/source/index.rst --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -6,13 +6,10 @@ code from Python, based on C-like declarations that you can often copy-paste from header files or documentation. -* Goals_ - - * `Comments and bugs`_ - .. toctree:: :maxdepth: 2 + goals whatsnew installation overview @@ -22,65 +19,4 @@ embedding -Goals ------ -The interface is based on `LuaJIT's FFI`_, and follows a few principles: - -* The goal is to call C code from Python without learning a 3rd language: - existing alternatives require users to learn domain specific language - (Cython_, SWIG_) or API (ctypes_). The CFFI design requires users to know - only C and Python, minimizing the extra bits of API that need to be learned. - -* Keep all the Python-related logic in Python so that you don't need to - write much C code (unlike `CPython native C extensions`_). - -* The preferred way is to work at the level of the API (Application - Programming Interface): the C compiler is called from the declarations - you write to validate and link to the C language constructs. - Alternatively, it is also possible to work at the ABI level - (Application Binary Interface), the way ctypes_ work. - However, on non-Windows platforms, C libraries typically - have a specified C API but not an ABI (e.g. they may - document a "struct" as having at least these fields, but maybe more). - -* Try to be complete. For now some C99 constructs are not supported, - but all C89 should be, including macros (and including macro "abuses", - which you can `manually wrap`_ in saner-looking C functions). - -* Attempt to support both PyPy and CPython, with a reasonable path - for other Python implementations like IronPython and Jython. - -* Note that this project is **not** about embedding executable C code in - Python, unlike `Weave`_. This is about calling existing C libraries - from Python. - -.. _`LuaJIT's FFI`: http://luajit.org/ext_ffi.html -.. _`Cython`: http://www.cython.org -.. _`SWIG`: http://www.swig.org/ -.. _`CPython native C extensions`: http://docs.python.org/extending/extending.html -.. _`native C extensions`: http://docs.python.org/extending/extending.html -.. _`ctypes`: http://docs.python.org/library/ctypes.html -.. _`Weave`: http://wiki.scipy.org/Weave -.. _`manually wrap`: overview.html#abi-versus-api - -Get started by reading `the overview`__. - -.. __: overview.html - - -Comments and bugs ------------------ - -The best way to contact us is on the IRC ``#pypy`` channel of -``irc.freenode.net``. Feel free to discuss matters either there or in -the `mailing list`_. Please report to the `issue tracker`_ any bugs. - -As a general rule, when there is a design issue to resolve, we pick the -solution that is the "most C-like". We hope that this module has got -everything you need to access C code and nothing more. - ---- the authors, Armin Rigo and Maciej Fijalkowski - -.. _`issue tracker`: https://bitbucket.org/cffi/cffi/issues -.. _`mailing list`: https://groups.google.com/forum/#!forum/python-cffi diff --git a/doc/source/installation.rst b/doc/source/installation.rst --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -53,13 +53,13 @@ * https://pypi.python.org/pypi/cffi -* Checksums of the "source" package version 1.11.3: +* Checksums of the "source" package version 1.11.4: - - MD5: 3d263ec84bb9ad08e0edf2d21312fa1e + - MD5: ... - - SHA: 26a73cc075064cc7cd59ec6c58ca0684d16c4ece + - SHA: ... - - SHA256: 150708ce9e417858f9a4056b5f364d8c7077fd979b9e35307c2d8a4a8e991fd2 + - SHA256: ... * Or grab the most current version from the `Bitbucket page`_: ``hg clone https://bitbucket.org/cffi/cffi`` diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst --- a/doc/source/whatsnew.rst +++ b/doc/source/whatsnew.rst @@ -2,6 +2,15 @@ What's New ====================== +v1.11.4 +======= + +* Windows: reverted linking with ``python3.dll``, because + virtualenv does not make this DLL available to virtual environments + for now. See `Issue #355`_. + +.. _`Issue #355`: https://bitbucket.org/cffi/cffi/issues/355/ + v1.11.3 ======= @@ -95,8 +104,11 @@ .. __: http://bugs.python.org/issue31105 +Older Versions +============== + v1.10.1 -======= +------- (only released inside PyPy 5.8.0) @@ -107,7 +119,7 @@ v1.10 -===== +----- * Issue #295: use calloc() directly instead of PyObject_Malloc()+memset() to handle ffi.new() with a default @@ -169,7 +181,7 @@ v1.9 -==== +---- * Structs with variable-sized arrays as their last field: now we track the length of the array after ``ffi.new()`` is called, just like we @@ -204,7 +216,7 @@ v1.8.3 -====== +------ * When passing a ``void *`` argument to a function with a different pointer type, or vice-versa, the cast occurs automatically, like in C. @@ -217,7 +229,7 @@ v1.8.2 -====== +------ * Issue #283: fixed ``ffi.new()`` on structures/unions with nested anonymous structures/unions, when there is at least one union in @@ -226,7 +238,7 @@ v1.8.1 -====== +------ * CPython 3.x: experimental: the generated C extension modules now use the "limited API", which means that, as a compiled .so/.dll, it should @@ -241,7 +253,7 @@ v1.8 -==== +---- * Removed the restriction that ``ffi.from_buffer()`` cannot be used on byte strings. Now you can get a ``char *`` out of a byte string, @@ -255,7 +267,7 @@ v1.7 -==== +---- * ``ffi.gc(p, None)`` removes the destructor on an object previously created by another call to ``ffi.gc()`` @@ -284,7 +296,7 @@ v1.6 -==== +---- * `ffi.list_types()`_ @@ -307,13 +319,13 @@ v1.5.2 -====== +------ * Fix 1.5.1 for Python 2.6. v1.5.1 -====== +------ * A few installation-time tweaks (thanks Stefano!) @@ -325,7 +337,7 @@ v1.5.0 -====== +------ * Support for `using CFFI for embedding`__. @@ -333,13 +345,13 @@ v1.4.2 -====== +------ Nothing changed from v1.4.1. v1.4.1 -====== +------ * Fix the compilation failure of cffi on CPython 3.5.0. (3.5.1 works; some detail changed that makes some underscore-starting macros @@ -349,7 +361,7 @@ v1.4.0 -====== +------ * A `better way to do callbacks`__ has been added (faster and more portable, and usually cleaner). It is a mechanism for the @@ -390,7 +402,7 @@ v1.3.1 -====== +------ * The optional typedefs (``bool``, ``FILE`` and all Windows types) were not always available from out-of-line FFI objects. @@ -406,7 +418,7 @@ v1.3.0 -====== +------ * Added `ffi.memmove()`_. @@ -441,13 +453,13 @@ v1.2.1 -====== +------ Nothing changed from v1.2.0. v1.2.0 -====== +------ * Out-of-line mode: ``int a[][...];`` can be used to declare a structure field or global variable which is, simultaneously, of total length @@ -500,14 +512,14 @@ v1.1.2 -====== +------ * ``ffi.gc()``: fixed a race condition in multithreaded programs introduced in 1.1.1 v1.1.1 -====== +------ * Out-of-line mode: ``ffi.string()``, ``ffi.buffer()`` and ``ffi.getwinerror()`` didn't accept their arguments as keyword @@ -525,7 +537,7 @@ v1.1.0 -====== +------ * Out-of-line API mode: we can now declare integer types with ``typedef int... foo_t;``. The exact size and signedness of ``foo_t`` @@ -558,13 +570,13 @@ v1.0.3 -====== +------ * Same as 1.0.2, apart from doc and test fixes on some platforms. v1.0.2 -====== +------ * Variadic C functions (ending in a "..." argument) were not supported in the out-of-line ABI mode. This was a bug---there was even a @@ -574,7 +586,7 @@ v1.0.1 -====== +------ * ``ffi.set_source()`` crashed if passed a ``sources=[..]`` argument. Fixed by chrippa on pull request #60. @@ -587,7 +599,7 @@ v1.0.0 -====== +------ * The main news item is out-of-line module generation: diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -186,7 +186,7 @@ `Mailing list `_ """, - version='1.11.3', + version='1.11.4', packages=['cffi'] if cpython else [], package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', '_embedding.h', '_cffi_errors.h']} From pypy.commits at gmail.com Sat Jan 13 08:17:08 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 13 Jan 2018 05:17:08 -0800 (PST) Subject: [pypy-commit] cffi default: Don't define py_limited_api at all on Windows, for now. Message-ID: <5a5a06d4.6c9adf0a.5e972.b20a@mx.google.com> Author: Armin Rigo Branch: Changeset: r3083:8444df673c93 Date: 2018-01-13 14:16 +0100 http://bitbucket.org/cffi/cffi/changeset/8444df673c93/ Log: Don't define py_limited_api at all on Windows, for now. diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h --- a/cffi/_cffi_include.h +++ b/cffi/_cffi_include.h @@ -15,7 +15,8 @@ version would not run inside a virtualenv. We will re-apply the fix after virtualenv has been fixed for some time. For explanation, see issue #355. For a workaround if you want PYTHON3.DLL and don't worry - about virtualenv, see issue #350. + about virtualenv, see issue #350. See also 'py_limited_api' in + setuptools_ext.py. */ #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) # include diff --git a/cffi/setuptools_ext.py b/cffi/setuptools_ext.py --- a/cffi/setuptools_ext.py +++ b/cffi/setuptools_ext.py @@ -81,8 +81,13 @@ it doesn't so far, creating troubles. That's why we check for "not hasattr(sys, 'gettotalrefcount')" (the 2.7 compatible equivalent of 'd' not in sys.abiflags). (http://bugs.python.org/issue28401) + + On Windows, it's better not to use py_limited_api until issue #355 + can be resolved (by having virtualenv copy PYTHON3.DLL). See also + the start of _cffi_include.h. """ - if 'py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount'): + if ('py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount') + and sys.platform != 'win32'): import setuptools try: setuptools_major_version = int(setuptools.__version__.partition('.')[0]) From pypy.commits at gmail.com Sat Jan 13 08:17:10 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 13 Jan 2018 05:17:10 -0800 (PST) Subject: [pypy-commit] cffi release-1.11: hg merge default Message-ID: <5a5a06d6.45a1df0a.a222c.8553@mx.google.com> Author: Armin Rigo Branch: release-1.11 Changeset: r3084:18a3d64c55a6 Date: 2018-01-13 14:16 +0100 http://bitbucket.org/cffi/cffi/changeset/18a3d64c55a6/ Log: hg merge default diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h --- a/cffi/_cffi_include.h +++ b/cffi/_cffi_include.h @@ -15,7 +15,8 @@ version would not run inside a virtualenv. We will re-apply the fix after virtualenv has been fixed for some time. For explanation, see issue #355. For a workaround if you want PYTHON3.DLL and don't worry - about virtualenv, see issue #350. + about virtualenv, see issue #350. See also 'py_limited_api' in + setuptools_ext.py. */ #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) # include diff --git a/cffi/setuptools_ext.py b/cffi/setuptools_ext.py --- a/cffi/setuptools_ext.py +++ b/cffi/setuptools_ext.py @@ -81,8 +81,13 @@ it doesn't so far, creating troubles. That's why we check for "not hasattr(sys, 'gettotalrefcount')" (the 2.7 compatible equivalent of 'd' not in sys.abiflags). (http://bugs.python.org/issue28401) + + On Windows, it's better not to use py_limited_api until issue #355 + can be resolved (by having virtualenv copy PYTHON3.DLL). See also + the start of _cffi_include.h. """ - if 'py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount'): + if ('py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount') + and sys.platform != 'win32'): import setuptools try: setuptools_major_version = int(setuptools.__version__.partition('.')[0]) From pypy.commits at gmail.com Sat Jan 13 11:24:47 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 13 Jan 2018 08:24:47 -0800 (PST) Subject: [pypy-commit] pypy.org extradoc: add s390x, regenerate Message-ID: <5a5a32cf.cb921c0a.c47fc.9170@mx.google.com> Author: Matti Picus Branch: extradoc Changeset: r913:dfcedac7e991 Date: 2018-01-13 18:24 +0200 http://bitbucket.org/pypy/pypy.org/changeset/dfcedac7e991/ Log: add s390x, regenerate diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -71,7 +71,7 @@ Linux, Mac OS X and Windows:

  • the Python2.7 compatible release — PyPy2.7 v5.10.0 — (what's new in PyPy2.7?)
    • -
  • the Python3.5 compatible beta quality release — PyPy3.5 v5.10.0 — (what's new in PyPy3.5?).
    • +
  • the Python3.5 compatible release — PyPy3.5 v5.10.1 — (what's new in PyPy3.5?).
  • the Python2.7 Software Transactional Memory special release — PyPy-STM 2.5.1 (Linux x86-64 only)
    @@ -130,21 +130,21 @@ mirror, but please use only if you have troubles accessing the links above
-
-

Python 3.5.3 compatible PyPy3.5 v5.10

+
+

Python 3.5.3 compatible PyPy3.5 v5.10.1

Warning: PyPy3.5 is known to be rarely much slower than PyPy 2. You are welcome to use it anyway

@@ -268,7 +268,7 @@

Alternatively, the following smaller package contains the source at the same revision as the above binaries:

  • Make sure you installed the dependencies. See the list here.

    @@ -410,12 +410,6 @@ 9afa1a36a5fc55ebc3e80576f05f44294f2b0de279862286fe00f5ee139965b1 pypy2-v5.10.0-ppc64.tar.bz2 2c32ccfa80e3e2ec56b4cc848526046d7b0de1f2f1a92b0cedeb414ec76745ab pypy2-v5.10.0-ppc64le.tar.bz2 -

    pypy 3.5-v5.9.0 sha256:

    -
    -d8c41ede3758127718944cc2fd6bf78ed4303d946f85596cac91281ccce36165  pypy3-v5.9.0-linux64.tar.bz2
-a014f47f50a1480f871a0b82705f904b38c93c4ca069850eb37653fedafb1b97  pypy3-v5.9.0-src.tar.bz2
-c5d7fa206cdf425de3950ef8ff578deb0745a723b69b64bb121210a5b8df8c65  pypy3-v5.9.0-src.zip
-

    pypy 3.5-v5.10.0 sha256:

     529bc3b11edbdcdd676d90c805b8f607f6eedd5f0ec457a31bbe09c03f5bebfe  pypy3-v5.10.0-linux32.tar.bz2
@@ -429,6 +423,18 @@
 96cf354fb410599cd5acd21732855e25e742e13eac7dc079c0c02b0625908cb9  pypy3-v5.10.0-src.zip
 2d93bf2bd7b1d031b96331d3fde6cacdda95673ce6875d6d1669c4c0ea2a52bc  pypy3-v5.10.0-win32.zip
    +

    pypy 3.5-v5.10.1 sha256:

    +
    +a6ceca9ee5dc511de7902164464b88311fec9366c5673d0c00528eda862bbe54  pypy3-v5.10.1-linux32.tar.bz2
+75a276e1ee1863967bbacb70c5bff636de200768c0ec90e72f7ec17aace0aefe  pypy3-v5.10.1-linux64.tar.bz2
+5065e9ad958d06b9612ba974f43997d20168d4245c054dd43270e4b458782282  pypy3-v5.10.1-linux-armel.tar.bz2
+203dd595fbad7055340b23326f20c85b0d6c11c4877e3559a437611fc2ac40c2  pypy3-v5.10.1-linux-armhf-raspbian.tar.bz2
+52f006611513c995fdebba6e72d394186d4085460408cbbe086e5467bf3fb9b6  pypy3-v5.10.1-osx64.tar.bz2
+f5548e06e2fc0c24ec8b6e3c5b09f90081818f7caa3e436dc312592611724713  pypy3-v5.10.1-src.tar.bz2
+182378d7aab395ee6cf539fb011ec0e384624282834aaaed4a663972a5aa8797  pypy3-v5.10.1-src.zip
+4edf4f021689a529e5a631c5cca72a1a9dc19a6ea2091e64289cdd5b60eaf929  pypy3-v5.10.1-win32.zip
+9ce98481cddede40a3357f7462f2c894bb96f178e2e8715d04feda1476ec1563  pypy3-v5.10.1-s390x.tar.bz2
+