[pypy-commit] pypy py3.5: hg merge default
arigo
pypy.commits at gmail.com
Sun Jun 11 17:22:45 EDT 2017
Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r91579:6e54776fdedf
Date: 2017-06-11 23:21 +0200
http://bitbucket.org/pypy/pypy/changeset/6e54776fdedf/
Log: hg merge default
diff too long, truncating to 2000 out of 4581 lines
diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -38,3 +38,5 @@
b16a4363e930f6401bceb499b9520955504c6cb0 release-pypy3.5-v5.7.0
1aa2d8e03cdfab54b7121e93fda7e98ea88a30bf release-pypy2.7-v5.7.1
2875f328eae2216a87f3d6f335092832eb031f56 release-pypy3.5-v5.7.1
+c925e73810367cd960a32592dd7f728f436c125c release-pypy2.7-v5.8.0
+a37ecfe5f142bc971a86d17305cc5d1d70abec64 release-pypy3.5-v5.8.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -39,11 +39,11 @@
Armin Rigo
Maciej Fijalkowski
- Carl Friedrich Bolz
+ Carl Friedrich Bolz-Tereick
Amaury Forgeot d'Arc
Antonio Cuni
+ Matti Picus
Samuele Pedroni
- Matti Picus
Ronan Lamy
Alex Gaynor
Philip Jenvey
@@ -101,28 +101,28 @@
Vincent Legoll
Michael Foord
Stephan Diehl
+ Stefano Rivera
Stefan Schwarzer
Tomek Meka
Valentino Volonghi
- Stefano Rivera
Patrick Maupin
Devin Jeanpierre
Bob Ippolito
Bruno Gola
David Malcolm
Jean-Paul Calderone
+ Squeaky
Edd Barrett
- Squeaky
Timo Paulssen
Marius Gedminas
Alexandre Fayolle
Simon Burton
Nicolas Truessel
Martin Matusiak
+ Laurence Tratt
Wenzhu Man
Konstantin Lopuhin
John Witulski
- Laurence Tratt
Greg Price
Ivan Sichmann Freitas
Dario Bertini
@@ -149,13 +149,13 @@
Stian Andreassen
Wanja Saatkamp
Mike Blume
+ Joannah Nanjekye
Gerald Klix
Oscar Nierstrasz
Rami Chowdhury
Stefan H. Muller
- Joannah Nanjekye
+ Tim Felgentreff
Eugene Oden
- Tim Felgentreff
Jeff Terrace
Henry Mason
Vasily Kuznetsov
@@ -164,11 +164,11 @@
Dusty Phillips
Lukas Renggli
Guenter Jantzen
+ Jasper Schulz
Ned Batchelder
Amit Regmi
Anton Gulenko
Sergey Matyunin
- Jasper Schulz
Andrew Chambers
Nicolas Chauvat
Andrew Durdin
@@ -183,6 +183,7 @@
Gintautas Miliauskas
Lucian Branescu Mihaila
anatoly techtonik
+ Dodan Mihai
Karl Bartel
Gabriel Lavoie
Jared Grubb
@@ -220,12 +221,14 @@
Vaibhav Sood
Reuben Cummings
Attila Gobi
+ Alecsandru Patrascu
Christopher Pope
Tristan Arthur
Christian Tismer
Dan Stromberg
Carl Meyer
Florin Papa
+ Jens-Uwe Mager
Valentina Mukhamedzhanova
Stefano Parmesan
touilleMan
@@ -264,7 +267,6 @@
Dan Buch
Lene Wagner
Tomo Cocoa
- Alecsandru Patrascu
David Lievens
Neil Blakey-Milner
Henrik Vendelbo
@@ -303,6 +305,7 @@
Anna Katrina Dominguez
Kim Jin Su
Amber Brown
+ Nate Bragg
Ben Darnell
Juan Francisco Cantero Hurtado
Godefroid Chappelle
@@ -340,11 +343,13 @@
Jim Hunziker
shoma hosaka
Buck Golemon
+ Iraklis D.
JohnDoe
yrttyr
Michael Chermside
Anna Ravencroft
remarkablerocket
+ Petre Vijiac
Berker Peksag
Christian Muirhead
soareschen
diff --git a/lib-python/2.7/warnings.py b/lib-python/2.7/warnings.py
--- a/lib-python/2.7/warnings.py
+++ b/lib-python/2.7/warnings.py
@@ -309,9 +309,12 @@
def __init__(self, message, category, filename, lineno, file=None,
line=None):
- local_values = locals()
- for attr in self._WARNING_DETAILS:
- setattr(self, attr, local_values[attr])
+ self.message = message
+ self.category = category
+ self.filename = filename
+ self.lineno = lineno
+ self.file = file
+ self.line = line
self._category_name = category.__name__ if category else None
def __str__(self):
diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -74,12 +74,16 @@
return self._type_._alignmentofinstances()
def _CData_output(self, resarray, base=None, index=-1):
- # this seems to be a string if we're array of char, surprise!
- from ctypes import c_char, c_wchar
- if self._type_ is c_char:
- return _rawffi.charp2string(resarray.buffer, self._length_)
- if self._type_ is c_wchar:
- return _rawffi.wcharp2unicode(resarray.buffer, self._length_)
+ from _rawffi.alt import types
+ # If a char_p or unichar_p is received, skip the string interpretation
+ if base._ffiargtype != types.Pointer(types.char_p) and \
+ base._ffiargtype != types.Pointer(types.unichar_p):
+ # this seems to be a string if we're array of char, surprise!
+ from ctypes import c_char, c_wchar
+ if self._type_ is c_char:
+ return _rawffi.charp2string(resarray.buffer, self._length_)
+ if self._type_ is c_wchar:
+ return _rawffi.wcharp2unicode(resarray.buffer, self._length_)
res = self.__new__(self)
ffiarray = self._ffiarray.fromaddress(resarray.buffer, self._length_)
res._buffer = ffiarray
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -159,9 +159,9 @@
#define _cffi_from_c_struct \
((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[18])
#define _cffi_to_c_wchar_t \
- ((wchar_t(*)(PyObject *))_cffi_exports[19])
+ ((_cffi_wchar_t(*)(PyObject *))_cffi_exports[19])
#define _cffi_from_c_wchar_t \
- ((PyObject *(*)(wchar_t))_cffi_exports[20])
+ ((PyObject *(*)(_cffi_wchar_t))_cffi_exports[20])
#define _cffi_to_c_long_double \
((long double(*)(PyObject *))_cffi_exports[21])
#define _cffi_to_c__Bool \
@@ -174,7 +174,11 @@
#define _CFFI_CPIDX 25
#define _cffi_call_python \
((void(*)(struct _cffi_externpy_s *, char *))_cffi_exports[_CFFI_CPIDX])
-#define _CFFI_NUM_EXPORTS 26
+#define _cffi_to_c_wchar3216_t \
+ ((int(*)(PyObject *))_cffi_exports[26])
+#define _cffi_from_c_wchar3216_t \
+ ((PyObject *(*)(int))_cffi_exports[27])
+#define _CFFI_NUM_EXPORTS 28
struct _cffi_ctypedescr;
@@ -215,6 +219,46 @@
return NULL;
}
+
+#ifdef HAVE_WCHAR_H
+typedef wchar_t _cffi_wchar_t;
+#else
+typedef uint16_t _cffi_wchar_t; /* same random pick as _cffi_backend.c */
+#endif
+
+_CFFI_UNUSED_FN static uint16_t _cffi_to_c_char16_t(PyObject *o)
+{
+ if (sizeof(_cffi_wchar_t) == 2)
+ return (uint16_t)_cffi_to_c_wchar_t(o);
+ else
+ return (uint16_t)_cffi_to_c_wchar3216_t(o);
+}
+
+_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char16_t(uint16_t x)
+{
+ if (sizeof(_cffi_wchar_t) == 2)
+ return _cffi_from_c_wchar_t(x);
+ else
+ return _cffi_from_c_wchar3216_t(x);
+}
+
+_CFFI_UNUSED_FN static int _cffi_to_c_char32_t(PyObject *o)
+{
+ if (sizeof(_cffi_wchar_t) == 4)
+ return (int)_cffi_to_c_wchar_t(o);
+ else
+ return (int)_cffi_to_c_wchar3216_t(o);
+}
+
+_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char32_t(int x)
+{
+ if (sizeof(_cffi_wchar_t) == 4)
+ return _cffi_from_c_wchar_t(x);
+ else
+ return _cffi_from_c_wchar3216_t(x);
+}
+
+
/********** end CPython-specific section **********/
#else
_CFFI_UNUSED_FN
diff --git a/lib_pypy/cffi/cffi_opcode.py b/lib_pypy/cffi/cffi_opcode.py
--- a/lib_pypy/cffi/cffi_opcode.py
+++ b/lib_pypy/cffi/cffi_opcode.py
@@ -107,9 +107,10 @@
PRIM_UINTMAX = 47
PRIM_FLOATCOMPLEX = 48
PRIM_DOUBLECOMPLEX = 49
+PRIM_CHAR16 = 50
+PRIM_CHAR32 = 51
-
-_NUM_PRIM = 50
+_NUM_PRIM = 52
_UNKNOWN_PRIM = -1
_UNKNOWN_FLOAT_PRIM = -2
_UNKNOWN_LONG_DOUBLE = -3
@@ -135,6 +136,8 @@
'double _Complex': PRIM_DOUBLECOMPLEX,
'_Bool': PRIM_BOOL,
'wchar_t': PRIM_WCHAR,
+ 'char16_t': PRIM_CHAR16,
+ 'char32_t': PRIM_CHAR32,
'int8_t': PRIM_INT8,
'uint8_t': PRIM_UINT8,
'int16_t': PRIM_INT16,
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -122,6 +122,8 @@
'_Bool': 'i',
# the following types are not primitive in the C sense
'wchar_t': 'c',
+ 'char16_t': 'c',
+ 'char32_t': 'c',
'int8_t': 'i',
'uint8_t': 'i',
'int16_t': 'i',
diff --git a/lib_pypy/cffi/parse_c_type.h b/lib_pypy/cffi/parse_c_type.h
--- a/lib_pypy/cffi/parse_c_type.h
+++ b/lib_pypy/cffi/parse_c_type.h
@@ -81,8 +81,10 @@
#define _CFFI_PRIM_UINTMAX 47
#define _CFFI_PRIM_FLOATCOMPLEX 48
#define _CFFI_PRIM_DOUBLECOMPLEX 49
+#define _CFFI_PRIM_CHAR16 50
+#define _CFFI_PRIM_CHAR32 51
-#define _CFFI__NUM_PRIM 50
+#define _CFFI__NUM_PRIM 52
#define _CFFI__UNKNOWN_PRIM (-1)
#define _CFFI__UNKNOWN_FLOAT_PRIM (-2)
#define _CFFI__UNKNOWN_LONG_DOUBLE (-3)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -3,8 +3,9 @@
from .error import VerificationError
from .cffi_opcode import *
-VERSION = "0x2601"
-VERSION_EMBEDDED = "0x2701"
+VERSION_BASE = 0x2601
+VERSION_EMBEDDED = 0x2701
+VERSION_CHAR16CHAR32 = 0x2801
class GlobalExpr:
@@ -126,6 +127,10 @@
self.ffi = ffi
self.module_name = module_name
self.target_is_python = target_is_python
+ self._version = VERSION_BASE
+
+ def needs_version(self, ver):
+ self._version = max(self._version, ver)
def collect_type_table(self):
self._typesdict = {}
@@ -304,9 +309,7 @@
prnt('#endif')
lines = self._rel_readlines('_embedding.h')
prnt(''.join(lines))
- version = VERSION_EMBEDDED
- else:
- version = VERSION
+ self.needs_version(VERSION_EMBEDDED)
#
# then paste the C source given by the user, verbatim.
prnt('/************************************************************/')
@@ -405,7 +408,7 @@
prnt(' _cffi_call_python_org = '
'(void(*)(struct _cffi_externpy_s *, char *))p[1];')
prnt(' }')
- prnt(' p[0] = (const void *)%s;' % version)
+ prnt(' p[0] = (const void *)0x%x;' % self._version)
prnt(' p[1] = &_cffi_type_context;')
prnt('}')
# on Windows, distutils insists on putting init_cffi_xyz in
@@ -423,21 +426,22 @@
prnt('PyMODINIT_FUNC')
prnt('PyInit_%s(void)' % (base_module_name,))
prnt('{')
- prnt(' return _cffi_init("%s", %s, &_cffi_type_context);' % (
- self.module_name, version))
+ prnt(' return _cffi_init("%s", 0x%x, &_cffi_type_context);' % (
+ self.module_name, self._version))
prnt('}')
prnt('#else')
prnt('PyMODINIT_FUNC')
prnt('init%s(void)' % (base_module_name,))
prnt('{')
- prnt(' _cffi_init("%s", %s, &_cffi_type_context);' % (
- self.module_name, version))
+ prnt(' _cffi_init("%s", 0x%x, &_cffi_type_context);' % (
+ self.module_name, self._version))
prnt('}')
prnt('#endif')
prnt()
prnt('#ifdef __GNUC__')
prnt('# pragma GCC visibility pop')
prnt('#endif')
+ self._version = None
def _to_py(self, x):
if isinstance(x, str):
@@ -476,7 +480,8 @@
prnt('from %s import ffi as _ffi%d' % (included_module_name, i))
prnt()
prnt("ffi = _cffi_backend.FFI('%s'," % (self.module_name,))
- prnt(" _version = %s," % (VERSION,))
+ prnt(" _version = 0x%x," % (self._version,))
+ self._version = None
#
# the '_types' keyword argument
self.cffi_types = tuple(self.cffi_types) # don't change any more
@@ -515,8 +520,11 @@
# double' here, and _cffi_to_c_double would loose precision
converter = '(%s)_cffi_to_c_double' % (tp.get_c_name(''),)
else:
- converter = '(%s)_cffi_to_c_%s' % (tp.get_c_name(''),
+ cname = tp.get_c_name('')
+ converter = '(%s)_cffi_to_c_%s' % (cname,
tp.name.replace(' ', '_'))
+ if cname in ('char16_t', 'char32_t'):
+ self.needs_version(VERSION_CHAR16CHAR32)
errvalue = '-1'
#
elif isinstance(tp, model.PointerType):
@@ -573,7 +581,10 @@
elif isinstance(tp, model.UnknownFloatType):
return '_cffi_from_c_double(%s)' % (var,)
elif tp.name != 'long double' and not tp.is_complex_type():
- return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var)
+ cname = tp.name.replace(' ', '_')
+ if cname in ('char16_t', 'char32_t'):
+ self.needs_version(VERSION_CHAR16CHAR32)
+ return '_cffi_from_c_%s(%s)' % (cname, var)
else:
return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % (
var, self._gettypenum(tp))
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -808,7 +808,8 @@
#include <stddef.h>
/* this block of #ifs should be kept exactly identical between
- c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py */
+ c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py
+ and cffi/_cffi_include.h */
#if defined(_MSC_VER)
# include <malloc.h> /* for alloca() */
# if _MSC_VER < 1600 /* MSVC < 2010 */
@@ -842,11 +843,13 @@
# include <stdint.h>
# endif
# if _MSC_VER < 1800 /* MSVC < 2013 */
- typedef unsigned char _Bool;
+# ifndef __cplusplus
+ typedef unsigned char _Bool;
+# endif
# endif
#else
# include <stdint.h>
-# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
# include <alloca.h>
# endif
#endif
diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py
--- a/lib_pypy/cffi/vengine_gen.py
+++ b/lib_pypy/cffi/vengine_gen.py
@@ -627,7 +627,8 @@
#include <sys/types.h> /* XXX for ssize_t on some platforms */
/* this block of #ifs should be kept exactly identical between
- c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py */
+ c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py
+ and cffi/_cffi_include.h */
#if defined(_MSC_VER)
# include <malloc.h> /* for alloca() */
# if _MSC_VER < 1600 /* MSVC < 2010 */
@@ -661,11 +662,13 @@
# include <stdint.h>
# endif
# if _MSC_VER < 1800 /* MSVC < 2013 */
- typedef unsigned char _Bool;
+# ifndef __cplusplus
+ typedef unsigned char _Bool;
+# endif
# endif
#else
# include <stdint.h>
-# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
# include <alloca.h>
# endif
#endif
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -195,6 +195,29 @@
``/tmp/usession-YOURNAME/build/``. You can then either move the file
hierarchy or unpack the ``.tar.bz2`` at the correct place.
+It is recommended to use package.py because custom scripts will
+invariably become out-of-date. If you want to write custom scripts
+anyway, note an easy-to-miss point: some modules are written with CFFI,
+and require some compilation. If you install PyPy as root without
+pre-compiling them, normal users will get errors:
+
+* PyPy 2.5.1 or earlier: normal users would see permission errors.
+ Installers need to run ``pypy -c "import gdbm"`` and other similar
+ commands at install time; the exact list is in `package.py`_. Users
+ seeing a broken installation of PyPy can fix it after-the-fact if they
+ have sudo rights, by running once e.g. ``sudo pypy -c "import gdbm``.
+
+* PyPy 2.6 and later: anyone would get ``ImportError: no module named
+ _gdbm_cffi``. Installers need to run ``pypy _gdbm_build.py`` in the
+ ``lib_pypy`` directory during the installation process (plus others;
+ see the exact list in `package.py`_). Users seeing a broken
+ installation of PyPy can fix it after-the-fact, by running ``pypy
+ /path/to/lib_pypy/_gdbm_build.py``. This command produces a file
+ called ``_gdbm_cffi.pypy-41.so`` locally, which is a C extension
+ module for PyPy. You can move it at any place where modules are
+ normally found: e.g. in your project's main directory, or in a
+ directory that you add to the env var ``PYTHONPATH``.
+
Installation
------------
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -59,16 +59,16 @@
# General information about the project.
project = u'PyPy'
-copyright = u'2016, The PyPy Project'
+copyright = u'2017, The PyPy Project'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
-version = '5.4'
+version = '5.8'
# The full version, including alpha/beta/rc tags.
-release = '5.4.0'
+release = '5.8.0'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -6,11 +6,11 @@
Armin Rigo
Maciej Fijalkowski
- Carl Friedrich Bolz
+ Carl Friedrich Bolz-Tereick
Amaury Forgeot d'Arc
Antonio Cuni
+ Matti Picus
Samuele Pedroni
- Matti Picus
Ronan Lamy
Alex Gaynor
Philip Jenvey
@@ -68,28 +68,28 @@
Vincent Legoll
Michael Foord
Stephan Diehl
+ Stefano Rivera
Stefan Schwarzer
Tomek Meka
Valentino Volonghi
- Stefano Rivera
Patrick Maupin
Devin Jeanpierre
Bob Ippolito
Bruno Gola
David Malcolm
Jean-Paul Calderone
+ Squeaky
Edd Barrett
- Squeaky
Timo Paulssen
Marius Gedminas
Alexandre Fayolle
Simon Burton
Nicolas Truessel
Martin Matusiak
+ Laurence Tratt
Wenzhu Man
Konstantin Lopuhin
John Witulski
- Laurence Tratt
Greg Price
Ivan Sichmann Freitas
Dario Bertini
@@ -116,13 +116,13 @@
Stian Andreassen
Wanja Saatkamp
Mike Blume
+ Joannah Nanjekye
Gerald Klix
Oscar Nierstrasz
Rami Chowdhury
Stefan H. Muller
- Joannah Nanjekye
+ Tim Felgentreff
Eugene Oden
- Tim Felgentreff
Jeff Terrace
Henry Mason
Vasily Kuznetsov
@@ -131,11 +131,11 @@
Dusty Phillips
Lukas Renggli
Guenter Jantzen
+ Jasper Schulz
Ned Batchelder
Amit Regmi
Anton Gulenko
Sergey Matyunin
- Jasper Schulz
Andrew Chambers
Nicolas Chauvat
Andrew Durdin
@@ -150,6 +150,7 @@
Gintautas Miliauskas
Lucian Branescu Mihaila
anatoly techtonik
+ Dodan Mihai
Karl Bartel
Gabriel Lavoie
Jared Grubb
@@ -187,12 +188,14 @@
Vaibhav Sood
Reuben Cummings
Attila Gobi
+ Alecsandru Patrascu
Christopher Pope
Tristan Arthur
Christian Tismer
Dan Stromberg
Carl Meyer
Florin Papa
+ Jens-Uwe Mager
Valentina Mukhamedzhanova
Stefano Parmesan
touilleMan
@@ -231,7 +234,6 @@
Dan Buch
Lene Wagner
Tomo Cocoa
- Alecsandru Patrascu
David Lievens
Neil Blakey-Milner
Henrik Vendelbo
@@ -270,6 +272,7 @@
Anna Katrina Dominguez
Kim Jin Su
Amber Brown
+ Nate Bragg
Ben Darnell
Juan Francisco Cantero Hurtado
Godefroid Chappelle
@@ -307,11 +310,13 @@
Jim Hunziker
shoma hosaka
Buck Golemon
+ Iraklis D.
JohnDoe
yrttyr
Michael Chermside
Anna Ravencroft
remarkablerocket
+ Petre Vijiac
Berker Peksag
Christian Muirhead
soareschen
diff --git a/pypy/doc/discussion/finalizer-order.rst b/pypy/doc/discussion/finalizer-order.rst
--- a/pypy/doc/discussion/finalizer-order.rst
+++ b/pypy/doc/discussion/finalizer-order.rst
@@ -60,7 +60,7 @@
The interface for full finalizers is made with PyPy in mind, but should
be generally useful.
-The idea is that you subclass the ``rgc.FinalizerQueue`` class::
+The idea is that you subclass the ``rgc.FinalizerQueue`` class:
* You must give a class-level attribute ``base_class``, which is the
base class of all instances with a finalizer. (If you need
diff --git a/pypy/doc/discussion/rawrefcount.rst b/pypy/doc/discussion/rawrefcount.rst
--- a/pypy/doc/discussion/rawrefcount.rst
+++ b/pypy/doc/discussion/rawrefcount.rst
@@ -68,10 +68,12 @@
and O = list of links created with rawrefcount.create_link_pyobj().
The PyPy objects in the list O are all W_CPyExtPlaceHolderObject: all
the data is in the PyObjects, and all outsite references (if any) are
-in C, as "PyObject *" fields.
+in C, as ``PyObject *`` fields.
So, during the collection we do this about P links:
+.. code-block:: python
+
for (p, ob) in P:
if ob->ob_refcnt != REFCNT_FROM_PYPY
and ob->ob_refcnt != REFCNT_FROM_PYPY_LIGHT:
@@ -80,6 +82,8 @@
At the end of the collection, the P and O links are both handled like
this:
+.. code-block:: python
+
for (p, ob) in P + O:
if p is not surviving: # even if 'ob' might be surviving
unlink p and ob
diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -5,8 +5,8 @@
++++++++++++++
We try to create a stable release a few times a year. These are released on
-a branch named like release-2.x or release-4.x, and each release is tagged,
-for instance release-4.0.1.
+a branch named like release-pypy3.5-v2.x or release-pypy3.5-v4.x, and each
+release is tagged, for instance release-pypy3.5-v4.0.1.
After release, inevitably there are bug fixes. It is the responsibility of
the commiter who fixes a bug to make sure this fix is on the release branch,
@@ -33,7 +33,7 @@
* If needed, make a release branch
* Bump the
pypy version number in module/sys/version.py and in
- module/cpyext/include/patchlevel.h and . The branch
+ module/cpyext/include/patchlevel.h and in doc/conf.py. The branch
will capture the revision number of this change for the release.
Some of the next updates may be done before or after branching; make
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -30,12 +30,22 @@
whatsnew-2.0.0-beta1.rst
whatsnew-1.9.rst
+CPython 3.5 compatible versions
+-------------------------------
+
+.. toctree::
+
+ whatsnew-pypy3-head.rst
+ whatsnew-pypy3-5.8.0.rst
+ whatsnew-pypy3-5.7.0.rst
+
CPython 3.3 compatible versions
-------------------------------
.. toctree::
whatsnew-pypy3-5.5.0.rst
+ whatsnew-pypy3-5.1.1-alpha1.rst
CPython 3.2 compatible versions
-------------------------------
diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst
--- a/pypy/doc/install.rst
+++ b/pypy/doc/install.rst
@@ -12,6 +12,7 @@
and using pip.
.. _prebuilt-pypy:
+
Download a pre-built PyPy
~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/pypy/doc/objspace.rst b/pypy/doc/objspace.rst
--- a/pypy/doc/objspace.rst
+++ b/pypy/doc/objspace.rst
@@ -250,12 +250,12 @@
.. py:function:: newunicode(ustr)
Creates a Unicode string from an rpython unicode string.
- This method may disappear soon and be replaced by :py:function:`newutf8()`.
+ This method may disappear soon and be replaced by :py:function::`newutf8`.
.. py:function:: newutf8(bytestr)
Creates a Unicode string from an rpython byte string, decoded as
- "utf-8-nosg". On PyPy3 it is the same as :py:function:`newtext()`.
+ "utf-8-nosg". On PyPy3 it is the same as :py:function::`newtext`.
Many more space operations can be found in `pypy/interpeter/baseobjspace.py` and
`pypy/objspace/std/objspace.py`.
@@ -302,9 +302,9 @@
.. py:function:: unicode_w(w_x)
- Takes an application level :py:class:`unicode` and return an
+ Takes an application level :py:class::`unicode` and return an
interpreter-level unicode string. This method may disappear soon and
- be replaced by :py:function:`text_w()`.
+ be replaced by :py:function::`text_w`.
.. py:function:: float_w(w_x)
diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
--- a/pypy/doc/release-pypy2.7-v5.4.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -171,7 +171,7 @@
* Performance improvements:
* Add a before_call()-like equivalent before a few operations like
- `malloc_nursery`, to move values from registers into other registers
+ `malloc_nursery`, to move values from registers into other registers
instead of to the stack.
* More tightly pack the stack when calling with `release gil`
diff --git a/pypy/doc/release-pypy2.7-v5.6.0.rst b/pypy/doc/release-pypy2.7-v5.6.0.rst
--- a/pypy/doc/release-pypy2.7-v5.6.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.6.0.rst
@@ -140,7 +140,7 @@
preamble
* In JIT residual calls, if the called function starts with a fast-path like
``if x.foo != 0: return x.foo``, then inline the check before doing the
- ``CALL``.
+ ``CALL``.
* Ensure ``make_inputargs`` fails properly when given arguments with type
information
* Makes ``optimiseopt`` iterative instead of recursive so it can be reasoned
diff --git a/pypy/doc/release-v5.8.0.rst b/pypy/doc/release-v5.8.0.rst
--- a/pypy/doc/release-v5.8.0.rst
+++ b/pypy/doc/release-v5.8.0.rst
@@ -3,33 +3,44 @@
=====================================
The PyPy team is proud to release both PyPy2.7 v5.8 (an interpreter supporting
-Python v2.7 syntax), and a beta-quality PyPy3.5 v5.8 (an interpreter for Python
-v3.5 syntax). The two releases are both based on much the same codebase, thus
+Python 2.7 syntax), and a beta-quality PyPy3.5 v5.8 (an interpreter for Python
+3.5 syntax). The two releases are both based on much the same codebase, thus
the dual release. Note that PyPy3.5 supports Linux 64bit only for now.
This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
PyPy3.5 includes the upstream stdlib version 3.5.3.
-This release enables `profile guided optimization` of the base interpreter,
-which may make unjitted code run faster.
+We fixed critical bugs in the shadowstack_ rootfinder garbage collector
+strategy that crashed multithreaded programs and very rarely showed up
+even in single threaded programs.
+
+We added native PyPy support to profile frames in the vmprof_ statistical
+profiler.
+
+The ``struct`` module functions ``pack*`` and ``unpack*`` are now much faster,
+especially on raw buffers and bytearrays. Microbenchmarks show a 2x to 10x
+speedup. Thanks to `Gambit Research`_ for sponsoring this work.
+
+This release adds (but disables by default) link-time optimization and
+`profile guided optimization`_ of the base interpreter, which may make
+unjitted code run faster. To use these, translate with appropriate
+`options`_. Be aware of `issues with gcc toolchains`_, though.
Please let us know if your use case is slow, we have ideas how to make things
faster but need real-world examples (not micro-benchmarks) of problematic code.
-Work proceeds at a good pace on the PyPy3.5
-version due to a grant_ from the Mozilla Foundation, hence our 3.5.3 beta
-release. Thanks Mozilla !!! While we do not pass all tests yet, asyncio works and
-as `these benchmarks show`_ it already gives a nice speed bump.
-We also backported the ``f""`` formatting from 3.6 (as an exception; otherwise
-"PyPy3.5" supports the Python 3.5 language).
+Work sponsored by a Mozilla grant_ continues on PyPy3.5; numerous fixes from
+CPython were ported to PyPy and PEP 489 was fully implemented. Of course the
+bug fixes and performance enhancements mentioned above are part of both PyPy
+2.7 and PyPy 3.5.
CFFI_, which is part of the PyPy release, has been updated to an unreleased 1.10.1,
improving an already great package for interfacing with C.
-As always, this release fixed many issues and bugs raised by the
+As always, this release fixed many other issues and bugs raised by the
growing community of PyPy users. We strongly recommend updating.
-You can download the v5.8 release here:
+You can download the v5.8 releases here:
http://pypy.org/download.html
@@ -43,13 +54,18 @@
with making RPython's JIT even better.
.. _`profile guided optimization`: https://pythonfiles.wordpress.com/2017/05/12/enabling-profile-guided-optimizations-for-pypy
+.. _shadowstack: config/translation.gcrootfinder.html
+.. _vmprof: http://vmprof.readthedocs.io
+.. _`issues with gcc toolchains`: https://bitbucket.org/pypy/pypy/issues/2572/link-time-optimization-lto-disabled
.. _CFFI: https://cffi.readthedocs.io/en/latest/whatsnew.html
.. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html
.. _`PyPy`: index.html
.. _`RPython`: https://rpython.readthedocs.org
.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly
.. _`help`: project-ideas.html
+.. _`options`: config/commandline.html#general-translation-options
.. _`these benchmarks show`: https://morepypy.blogspot.com/2017/03/async-http-benchmarks-on-pypy3.html
+.. _`Gambit Research`: http://gambitresearch.com
What is PyPy?
=============
@@ -148,7 +164,7 @@
accepted in a few more places, e.g. in compile()
-.. _here: http://rpython.readthedocs.io/en/latest/cpython_differences.html
+.. _here: cpython_differences.html
Highlights of the PyPy3.5 release (since 5.7 beta released March 2017)
======================================================================
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
--- a/pypy/doc/tool/makecontributor.py
+++ b/pypy/doc/tool/makecontributor.py
@@ -18,8 +18,9 @@
'Antonio Cuni': ['antocuni', 'anto'],
'Armin Rigo': ['arigo', 'arfigo', 'armin', 'arigato'],
'Maciej Fijalkowski': ['fijal'],
- 'Carl Friedrich Bolz': ['cfbolz', 'cf'],
+ 'Carl Friedrich Bolz-Tereick': ['Carl Friedrich Bolz', 'cfbolz', 'cf'],
'Samuele Pedroni': ['pedronis', 'samuele', 'samule'],
+ 'Richard Plangger':['planrich'],
'Michael Hudson': ['mwh'],
'Holger Krekel': ['hpk', 'holger krekel', 'holger', 'hufpk'],
"Amaury Forgeot d'Arc": ['afa'],
@@ -67,7 +68,7 @@
'Edd Barrett': ['edd'],
'Manuel Jacob': ['mjacob'],
'Rami Chowdhury': ['necaris'],
- 'Stanislaw Halik':['w31rd0'],
+ 'Stanislaw Halik': ['Stanislaw Halik', 'w31rd0'],
'Wenzhu Man':['wenzhu man', 'wenzhuman'],
'Anton Gulenko':['anton gulenko', 'anton_gulenko'],
'Richard Lancaster':['richardlancaster'],
@@ -78,7 +79,8 @@
'Jasper Schulz':['Jasper.Schulz', 'jbs'],
'Aaron Gallagher':['"Aaron Gallagher'],
'Yasir Suhail':['yasirs'],
- 'Squeaky', ['squeaky'],
+ 'Squeaky': ['squeaky'],
+ "Amaury Forgeot d'Arc": ['amauryfa at gmail.com'],
}
alias_map = {}
diff --git a/pypy/doc/whatsnew-2.6.1.rst b/pypy/doc/whatsnew-2.6.1.rst
--- a/pypy/doc/whatsnew-2.6.1.rst
+++ b/pypy/doc/whatsnew-2.6.1.rst
@@ -6,6 +6,7 @@
.. startrev: 91904d5c5188
.. branch: use_min_scalar
+
Correctly resolve the output dtype of ufunc(array, scalar) calls.
.. branch: stdlib-2.7.10
@@ -15,6 +16,7 @@
.. branch: issue2062
.. branch: disable-unroll-for-short-loops
+
The JIT no longer performs loop unrolling if the loop compiles to too much code.
.. branch: run-create_cffi_imports
@@ -32,9 +34,11 @@
``lst[0]`` is still *not* the float ``42.0`` but the integer ``42``.)
.. branch: cffi-callback-onerror
+
Part of cffi 1.2.
.. branch: cffi-new-allocator
+
Part of cffi 1.2.
.. branch: unicode-dtype
diff --git a/pypy/doc/whatsnew-4.0.0.rst b/pypy/doc/whatsnew-4.0.0.rst
--- a/pypy/doc/whatsnew-4.0.0.rst
+++ b/pypy/doc/whatsnew-4.0.0.rst
@@ -6,23 +6,28 @@
.. startrev: 3a8f5481dab4
.. branch: keys_with_hash
+
Improve the performance of ``dict.update()`` and a bunch of methods from
sets, by reusing the hash value stored in one dict when inspecting
or changing another dict with that key.
.. branch: optresult-unroll
+
A major refactoring of the ``ResOperations`` that kills Box. Also rewrote
unrolling to enable future enhancements. Should improve warmup time
by 20% or so.
.. branch: optimize-cond-call
+
Optimize common sequences of operations like
``int_lt/cond_call`` in the JIT backends
.. branch: missing_openssl_include
+
Fix for missing headers in OpenBSD, already applied in downstream ports
.. branch: gc-more-incremental
+
Remove a source of non-incremental-ness in the GC: now
``external_malloc()`` no longer runs ``gc_step_until()`` any more. If there
is a currently-running major collection, we do only so many steps
@@ -32,11 +37,13 @@
keep adding up between them.
.. branch: remember-tracing-counts
+
Reenable jithooks
.. branch: detect_egd2
.. branch: shadowstack-no-move-2
+
Issue #2141: fix a crash on Windows and OS/X and ARM when running
at least 20 threads.
@@ -55,6 +62,7 @@
floats, cf. issue #2148.
.. branch: cffi-stdcall
+
Win32: support ``__stdcall`` in CFFI.
.. branch: callfamily
@@ -93,6 +101,7 @@
.. branch: osx-libffi
.. branch: lazy-fast2locals
-improve the performance of simple trace functions by lazily calling
+
+Improve the performance of simple trace functions by lazily calling
``fast2locals`` and ``locals2fast`` only if ``f_locals`` is actually accessed.
diff --git a/pypy/doc/whatsnew-5.0.0.rst b/pypy/doc/whatsnew-5.0.0.rst
--- a/pypy/doc/whatsnew-5.0.0.rst
+++ b/pypy/doc/whatsnew-5.0.0.rst
@@ -192,6 +192,7 @@
Fix boolean-array indexing in micronumpy
.. branch: numpy_partition
+
Support ndarray.partition() as an app-level function numpy.core._partition_use,
provided as a cffi wrapper to upstream's implementation in the pypy/numpy repo
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -6,5 +6,6 @@
.. startrev: 558bd00b3dd8
.. branch: cffi-complex
+.. branch: cffi-char16-char32
-Part of the upgrade to cffi 1.11
+The two ``cffi-*`` branches are part of the upgrade to cffi 1.11.
diff --git a/pypy/doc/whatsnew-pypy2-5.3.0.rst b/pypy/doc/whatsnew-pypy2-5.3.0.rst
--- a/pypy/doc/whatsnew-pypy2-5.3.0.rst
+++ b/pypy/doc/whatsnew-pypy2-5.3.0.rst
@@ -29,6 +29,7 @@
upstream numpy via cpyext, so we created (yet another) fork of numpy at
github.com/pypy/numpy with the needed changes. Among the significant changes
to cpyext:
+
- allow c-snippet tests to be run with -A so we can verify we are compatible
- fix many edge cases exposed by fixing tests to run with -A
- issequence() logic matches cpython
diff --git a/pypy/doc/whatsnew-pypy2-5.4.0.rst b/pypy/doc/whatsnew-pypy2-5.4.0.rst
--- a/pypy/doc/whatsnew-pypy2-5.4.0.rst
+++ b/pypy/doc/whatsnew-pypy2-5.4.0.rst
@@ -6,10 +6,12 @@
.. startrev: 873218a739f1
.. 418b05f95db5
+
Improve CPython compatibility for ``is``. Now code like ``if x is ():``
works the same way as it does on CPython. See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
.. pull request #455
+
Add sys.{get,set}dlopenflags, for cpyext extensions.
.. branch: fix-gen-dfa
@@ -36,9 +38,11 @@
compatible.
.. branch: pyfile-tell
+
Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
.. branch: rw-PyString_AS_STRING
+
Allow rw access to the char* returned from PyString_AS_STRING, also refactor
PyStringObject to look like cpython's and allow subclassing PyString_Type and
PyUnicode_Type
diff --git a/pypy/doc/whatsnew-pypy2-5.6.0.rst b/pypy/doc/whatsnew-pypy2-5.6.0.rst
--- a/pypy/doc/whatsnew-pypy2-5.6.0.rst
+++ b/pypy/doc/whatsnew-pypy2-5.6.0.rst
@@ -12,18 +12,22 @@
.. branch: testing-cleanup-py3k
.. branch: rpython-resync
+
Backport rpython changes made directly on the py3k and py3.5 branches.
.. branch: buffer-interface
+
Implement PyObject_GetBuffer, PyMemoryView_GET_BUFFER, and handles memoryviews
in numpypy
.. branch: force-virtual-state
+
Improve merging of virtual states in the JIT in order to avoid jumping to the
preamble. Accomplished by allocating virtual objects where non-virtuals are
expected.
.. branch: conditional_call_value_3
+
JIT residual calls: if the called function starts with a fast-path
like "if x.foo != 0: return x.foo", then inline the check before
doing the CALL. For now, string hashing is about the only case.
@@ -64,6 +68,7 @@
.. fb6bb835369e
+
Change the ``timeit`` module: it now prints the average time and the standard
deviation over 7 runs by default, instead of the minimum. The minimum is often
misleading.
@@ -75,9 +80,6 @@
.. branch: Tiberiumk/fix-2412-1476011166874
.. branch: redirect-assembler-jitlog
-
-
-
.. branch: stdlib-2.7.12
Update stdlib to version 2.7.12
diff --git a/pypy/doc/whatsnew-pypy3-5.8.0.rst b/pypy/doc/whatsnew-pypy3-5.8.0.rst
--- a/pypy/doc/whatsnew-pypy3-5.8.0.rst
+++ b/pypy/doc/whatsnew-pypy3-5.8.0.rst
@@ -6,6 +6,7 @@
.. startrev: afbf09453369
.. branch: mtest
+
Use "<python> -m test" to run the CPython test suite, as documented by CPython,
instead of our outdated regrverbose.py script.
diff --git a/pypy/module/_cffi_backend/cffi1_module.py b/pypy/module/_cffi_backend/cffi1_module.py
--- a/pypy/module/_cffi_backend/cffi1_module.py
+++ b/pypy/module/_cffi_backend/cffi1_module.py
@@ -9,7 +9,7 @@
VERSION_MIN = 0x2601
-VERSION_MAX = 0x27FF
+VERSION_MAX = 0x28FF
VERSION_EXPORT = 0x0A03
diff --git a/pypy/module/_cffi_backend/cffi_opcode.py b/pypy/module/_cffi_backend/cffi_opcode.py
--- a/pypy/module/_cffi_backend/cffi_opcode.py
+++ b/pypy/module/_cffi_backend/cffi_opcode.py
@@ -107,8 +107,10 @@
PRIM_UINTMAX = 47
PRIM_FLOATCOMPLEX = 48
PRIM_DOUBLECOMPLEX = 49
+PRIM_CHAR16 = 50
+PRIM_CHAR32 = 51
-_NUM_PRIM = 50
+_NUM_PRIM = 52
_UNKNOWN_PRIM = -1
_UNKNOWN_FLOAT_PRIM = -2
_UNKNOWN_LONG_DOUBLE = -3
@@ -131,8 +133,12 @@
'float': PRIM_FLOAT,
'double': PRIM_DOUBLE,
'long double': PRIM_LONGDOUBLE,
+ 'float _Complex': PRIM_FLOATCOMPLEX,
+ 'double _Complex': PRIM_DOUBLECOMPLEX,
'_Bool': PRIM_BOOL,
'wchar_t': PRIM_WCHAR,
+ 'char16_t': PRIM_CHAR16,
+ 'char32_t': PRIM_CHAR32,
'int8_t': PRIM_INT8,
'uint8_t': PRIM_UINT8,
'int16_t': PRIM_INT16,
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -36,8 +36,7 @@
datasize = self.size
#
if datasize < 0:
- from pypy.module._cffi_backend import misc
- w_init, length = misc.get_new_array_length(space, w_init)
+ w_init, length = self.get_new_array_length(w_init)
try:
datasize = ovfcheck(length * self.ctitem.size)
except OverflowError:
@@ -53,6 +52,29 @@
self.convert_from_object(ptr, w_init)
return cdata
+ def get_new_array_length(self, w_value):
+ space = self.space
+ if (space.isinstance_w(w_value, space.w_list) or
+ space.isinstance_w(w_value, space.w_tuple)):
+ return (w_value, space.int_w(space.len(w_value)))
+ elif space.isinstance_w(w_value, space.w_bytes):
+ # from a string, we add the null terminator
+ s = space.bytes_w(w_value)
+ return (w_value, len(s) + 1)
+ elif space.isinstance_w(w_value, space.w_unicode):
+ from pypy.module._cffi_backend import wchar_helper
+ u = space.unicode_w(w_value)
+ if self.ctitem.size == 2:
+ length = wchar_helper.unicode_size_as_char16(u)
+ else:
+ length = wchar_helper.unicode_size_as_char32(u)
+ return (w_value, length + 1)
+ else:
+ explicitlength = space.getindex_w(w_value, space.w_OverflowError)
+ if explicitlength < 0:
+ raise oefmt(space.w_ValueError, "negative array length")
+ return (space.w_None, explicitlength)
+
def _check_subscript_index(self, w_cdata, i):
space = self.space
if i < 0:
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -10,7 +10,7 @@
from rpython.rtyper.tool import rfficache
from pypy.interpreter.error import oefmt
-from pypy.module._cffi_backend import cdataobj, misc
+from pypy.module._cffi_backend import cdataobj, misc, wchar_helper
from pypy.module._cffi_backend.ctypeobj import W_CType
@@ -42,11 +42,13 @@
def cast_unicode(self, w_ob):
space = self.space
s = space.unicode_w(w_ob)
- if len(s) != 1:
+ try:
+ ordinal = wchar_helper.unicode_to_ordinal(s)
+ except ValueError:
raise oefmt(space.w_TypeError,
"cannot cast unicode string of length %d to ctype '%s'",
len(s), self.name)
- return ord(s[0])
+ return intmask(ordinal)
def cast(self, w_ob):
from pypy.module._cffi_backend import ctypeptr
@@ -148,53 +150,83 @@
return self.space.newbytes(s)
-# XXX explicitly use an integer type instead of lltype.UniChar here,
-# because for now the latter is defined as unsigned by RPython (even
-# though it may be signed when 'wchar_t' is written to C).
-WCHAR_INT = {(2, False): rffi.USHORT,
- (4, False): rffi.UINT,
- (4, True): rffi.INT}[rffi.sizeof(lltype.UniChar),
- rfficache.signof_c_type('wchar_t')]
-WCHAR_INTP = rffi.CArrayPtr(WCHAR_INT)
+class W_CTypePrimitiveUniChar(W_CTypePrimitiveCharOrUniChar):
+ _attrs_ = ['is_signed_wchar']
+ _immutable_fields_ = ['is_signed_wchar']
-class W_CTypePrimitiveUniChar(W_CTypePrimitiveCharOrUniChar):
- _attrs_ = []
+ _wchar_is_signed = rfficache.signof_c_type('wchar_t')
- if rffi.r_wchar_t.SIGN:
- def write_raw_integer_data(self, w_cdata, value):
- w_cdata.write_raw_signed_data(value)
+ def __init__(self, space, size, name, name_position, align):
+ W_CTypePrimitiveCharOrUniChar.__init__(self, space, size, name,
+ name_position, align)
+ self.is_signed_wchar = self._wchar_is_signed and (name == "wchar_t")
+ # "char16_t" and "char32_t" are always unsigned
def cast_to_int(self, cdata):
- unichardata = rffi.cast(WCHAR_INTP, cdata)
- return self.space.newint(unichardata[0])
+ if self.is_signed_wchar:
+ value = misc.read_raw_long_data(cdata, self.size)
+ return self.space.newint(value)
+ else:
+ value = misc.read_raw_ulong_data(cdata, self.size)
+ if self.size < rffi.sizeof(lltype.Signed):
+ return self.space.newint(intmask(value))
+ else:
+ return self.space.newint(value) # r_uint => 'long' object
def convert_to_object(self, cdata):
- unichardata = rffi.cast(rffi.CWCHARP, cdata)
- return self.space.newunicode(unichardata[0])
+ if self.is_signed_wchar:
+ unichardata = rffi.cast(rffi.CWCHARP, cdata)
+ return self.space.newunicode(unichardata[0])
+ else:
+ value = misc.read_raw_ulong_data(cdata, self.size) # r_uint
+ try:
+ u = wchar_helper.ordinal_to_unicode(value)
+ except wchar_helper.OutOfRange as e:
+ raise oefmt(self.space.w_ValueError,
+ "char32_t out of range for "
+ "conversion to unicode: %s", hex(e.ordinal))
+ return self.space.newunicode(u)
def string(self, cdataobj, maxlen):
with cdataobj as ptr:
w_res = self.convert_to_object(ptr)
return w_res
- def _convert_to_unichar(self, w_ob):
+ def _convert_to_charN_t(self, w_ob):
+ # returns a r_uint. If self.size == 2, it is smaller than 0x10000
space = self.space
if space.isinstance_w(w_ob, space.w_unicode):
- s = space.unicode_w(w_ob)
- if len(s) == 1:
- return s[0]
- if (isinstance(w_ob, cdataobj.W_CData) and
- isinstance(w_ob.ctype, W_CTypePrimitiveUniChar)):
+ u = space.unicode_w(w_ob)
+ try:
+ ordinal = wchar_helper.unicode_to_ordinal(u)
+ except ValueError:
+ pass
+ else:
+ if self.size == 2 and ordinal > 0xffff:
+ raise self._convert_error("single character <= 0xFFFF",
+ w_ob)
+ return ordinal
+ elif (isinstance(w_ob, cdataobj.W_CData) and
+ isinstance(w_ob.ctype, W_CTypePrimitiveUniChar) and
+ w_ob.ctype.size == self.size):
with w_ob as ptr:
- return rffi.cast(rffi.CWCHARP, ptr)[0]
+ return misc.read_raw_ulong_data(ptr, self.size)
raise self._convert_error("unicode string of length 1", w_ob)
def convert_from_object(self, cdata, w_ob):
- value = self._convert_to_unichar(w_ob)
- rffi.cast(rffi.CWCHARP, cdata)[0] = value
+ ordinal = self._convert_to_charN_t(w_ob)
+ misc.write_raw_unsigned_data(cdata, ordinal, self.size)
def unpack_ptr(self, w_ctypeptr, ptr, length):
- u = rffi.wcharpsize2unicode(rffi.cast(rffi.CWCHARP, ptr), length)
+ if self.size == 2:
+ u = wchar_helper.unicode_from_char16(ptr, length)
+ else:
+ try:
+ u = wchar_helper.unicode_from_char32(ptr, length)
+ except wchar_helper.OutOfRange as e:
+ raise oefmt(self.space.w_ValueError,
+ "char32_t out of range for "
+ "conversion to unicode: %s", hex(e.ordinal))
return self.space.newunicode(u)
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -6,9 +6,9 @@
from rpython.rlib import rposix
from rpython.rlib.rarithmetic import ovfcheck
-from rpython.rtyper.annlowlevel import llstr, llunicode
+from rpython.rtyper.annlowlevel import llstr
from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw, copy_unicode_to_raw
+from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
from pypy.module._cffi_backend import cdataobj, misc, ctypeprim, ctypevoid
@@ -90,18 +90,28 @@
if n != self.length:
cdata[n] = '\x00'
elif isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar):
+ from pypy.module._cffi_backend import wchar_helper
if not space.isinstance_w(w_ob, space.w_unicode):
raise self._convert_error("unicode or list or tuple", w_ob)
s = space.unicode_w(w_ob)
- n = len(s)
+ if self.ctitem.size == 2:
+ n = wchar_helper.unicode_size_as_char16(s)
+ else:
+ n = wchar_helper.unicode_size_as_char32(s)
if self.length >= 0 and n > self.length:
raise oefmt(space.w_IndexError,
"initializer unicode string is too long for '%s' "
"(got %d characters)", self.name, n)
- unichardata = rffi.cast(rffi.CWCHARP, cdata)
- copy_unicode_to_raw(llunicode(s), unichardata, 0, n)
- if n != self.length:
- unichardata[n] = u'\x00'
+ add_final_zero = (n != self.length)
+ if self.ctitem.size == 2:
+ try:
+ wchar_helper.unicode_to_char16(s, cdata, n, add_final_zero)
+ except wchar_helper.OutOfRange as e:
+ raise oefmt(self.space.w_ValueError,
+ "unicode character ouf of range for "
+ "conversion to char16_t: %s", hex(e.ordinal))
+ else:
+ wchar_helper.unicode_to_char32(s, cdata, n, add_final_zero)
else:
raise self._convert_error("list or tuple", w_ob)
@@ -136,12 +146,12 @@
#
# pointer to a wchar_t: builds and returns a unicode
if self.is_unichar_ptr_or_array():
- cdata = rffi.cast(rffi.CWCHARP, ptr)
- if length < 0:
- u = rffi.wcharp2unicode(cdata)
+ from pypy.module._cffi_backend import wchar_helper
+ if self.ctitem.size == 2:
+ length = wchar_helper.measure_length_16(ptr, length)
else:
- u = rffi.wcharp2unicoden(cdata, length)
- return space.newunicode(u)
+ length = wchar_helper.measure_length_32(ptr, length)
+ return self.ctitem.unpack_ptr(self, ptr, length)
#
return W_CType.string(self, cdataobj, maxlen)
@@ -304,10 +314,18 @@
if (space.isinstance_w(w_init, space.w_list) or
space.isinstance_w(w_init, space.w_tuple)):
length = space.int_w(space.len(w_init))
- elif (space.isinstance_w(w_init, space.w_unicode) or
- space.isinstance_w(w_init, space.w_bytes)):
+ elif space.isinstance_w(w_init, space.w_bytes):
# from a string, we add the null terminator
- length = space.int_w(space.len(w_init)) + 1
+ s = space.bytes_w(w_init)
+ length = len(s) + 1
+ elif space.isinstance_w(w_init, space.w_unicode):
+ from pypy.module._cffi_backend import wchar_helper
+ u = space.unicode_w(w_init)
+ if self.ctitem.size == 2:
+ length = wchar_helper.unicode_size_as_char16(u)
+ else:
+ length = wchar_helper.unicode_size_as_char32(u)
+ length += 1
elif self.is_file:
result = self.prepare_file(w_init)
if result:
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -244,7 +244,7 @@
ct = self.ctype
if isinstance(ct, ctypearray.W_CTypeArray) and ct.length < 0:
space = ct.space
- w_ob, varsizelength = misc.get_new_array_length(space, w_ob)
+ w_ob, varsizelength = ct.get_new_array_length(w_ob)
if optvarsize != -1:
# in this mode, the only purpose of this function is to compute
# the real size of the structure from a var-sized C99 array
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -294,22 +294,6 @@
# ____________________________________________________________
-def get_new_array_length(space, w_value):
- if (space.isinstance_w(w_value, space.w_list) or
- space.isinstance_w(w_value, space.w_tuple)):
- return (w_value, space.int_w(space.len(w_value)))
- elif (space.isinstance_w(w_value, space.w_unicode) or
- space.isinstance_w(w_value, space.w_bytes)):
- # from a string, we add the null terminator
- return (w_value, space.int_w(space.len(w_value)) + 1)
- else:
- explicitlength = space.getindex_w(w_value, space.w_OverflowError)
- if explicitlength < 0:
- raise oefmt(space.w_ValueError, "negative array length")
- return (space.w_None, explicitlength)
-
-# ____________________________________________________________
-
@specialize.arg(0)
def _raw_memcopy_tp(TPP, source, dest):
# in its own function: LONGLONG may make the whole function jit-opaque
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -111,6 +111,9 @@
eptype("size_t", rffi.SIZE_T, ctypeprim.W_CTypePrimitiveUnsigned)
eptype("ssize_t", rffi.SSIZE_T, ctypeprim.W_CTypePrimitiveSigned)
+eptypesize("char16_t", 2, ctypeprim.W_CTypePrimitiveUniChar)
+eptypesize("char32_t", 4, ctypeprim.W_CTypePrimitiveUniChar)
+
_WCTSigned = ctypeprim.W_CTypePrimitiveSigned
_WCTUnsign = ctypeprim.W_CTypePrimitiveUnsigned
diff --git a/pypy/module/_cffi_backend/realize_c_type.py b/pypy/module/_cffi_backend/realize_c_type.py
--- a/pypy/module/_cffi_backend/realize_c_type.py
+++ b/pypy/module/_cffi_backend/realize_c_type.py
@@ -73,6 +73,8 @@
"uintmax_t",
"float _Complex",
"double _Complex",
+ "char16_t",
+ "char32_t",
]
assert len(NAMES) == cffi_opcode._NUM_PRIM
diff --git a/pypy/module/_cffi_backend/src/parse_c_type.c b/pypy/module/_cffi_backend/src/parse_c_type.c
--- a/pypy/module/_cffi_backend/src/parse_c_type.c
+++ b/pypy/module/_cffi_backend/src/parse_c_type.c
@@ -505,6 +505,7 @@
case '1':
if (size == 8 && !memcmp(p, "uint16", 6)) return _CFFI_PRIM_UINT16;
+ if (size == 8 && !memcmp(p, "char16", 6)) return _CFFI_PRIM_CHAR16;
break;
case '2':
@@ -513,6 +514,7 @@
case '3':
if (size == 8 && !memcmp(p, "uint32", 6)) return _CFFI_PRIM_UINT32;
+ if (size == 8 && !memcmp(p, "char32", 6)) return _CFFI_PRIM_CHAR32;
break;
case '4':
diff --git a/pypy/module/_cffi_backend/src/parse_c_type.h b/pypy/module/_cffi_backend/src/parse_c_type.h
--- a/pypy/module/_cffi_backend/src/parse_c_type.h
+++ b/pypy/module/_cffi_backend/src/parse_c_type.h
@@ -80,8 +80,10 @@
#define _CFFI_PRIM_UINTMAX 47
#define _CFFI_PRIM_FLOATCOMPLEX 48
#define _CFFI_PRIM_DOUBLECOMPLEX 49
+#define _CFFI_PRIM_CHAR16 50
+#define _CFFI_PRIM_CHAR32 51
-#define _CFFI__NUM_PRIM 50
+#define _CFFI__NUM_PRIM 52
#define _CFFI__UNKNOWN_PRIM (-1)
#define _CFFI__UNKNOWN_FLOAT_PRIM (-2)
#define _CFFI__UNKNOWN_LONG_DOUBLE (-3)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1925,7 +1925,11 @@
assert string(a, 8).startswith(b'ABC') # may contain additional garbage
def test_string_wchar():
- BWChar = new_primitive_type("wchar_t")
+ for typename in ["wchar_t", "char16_t", "char32_t"]:
+ _test_string_wchar_variant(typename)
+
+def _test_string_wchar_variant(typename):
+ BWChar = new_primitive_type(typename)
assert string(cast(BWChar, 42)) == u+'*'
assert string(cast(BWChar, 0x4253)) == u+'\u4253'
assert string(cast(BWChar, 0)) == u+'\x00'
@@ -2087,22 +2091,44 @@
py.test.raises(TypeError, newp, BStructPtr, [cast(BFunc2, 0)])
def test_wchar():
- BWChar = new_primitive_type("wchar_t")
+ _test_wchar_variant("wchar_t")
+ if sys.platform.startswith("linux"):
+ BWChar = new_primitive_type("wchar_t")
+ assert sizeof(BWChar) == 4
+ assert int(cast(BWChar, -1)) == -1 # signed, on linux
+
+def test_char16():
+ BChar16 = new_primitive_type("char16_t")
+ assert sizeof(BChar16) == 2
+ _test_wchar_variant("char16_t")
+ assert int(cast(BChar16, -1)) == 0xffff # always unsigned
+
+def test_char32():
+ BChar32 = new_primitive_type("char32_t")
+ assert sizeof(BChar32) == 4
+ _test_wchar_variant("char32_t")
+ assert int(cast(BChar32, -1)) == 0xffffffff # always unsigned
+
+def _test_wchar_variant(typename):
+ BWChar = new_primitive_type(typename)
BInt = new_primitive_type("int")
pyuni4 = {1: True, 2: False}[len(u+'\U00012345')]
wchar4 = {2: False, 4: True}[sizeof(BWChar)]
- assert str(cast(BWChar, 0x45)) == "<cdata 'wchar_t' %s'E'>" % (
- mandatory_u_prefix,)
- assert str(cast(BWChar, 0x1234)) == "<cdata 'wchar_t' %s'\u1234'>" % (
- mandatory_u_prefix,)
- if wchar4:
- if not _hacked_pypy_uni4():
+ assert str(cast(BWChar, 0x45)) == "<cdata '%s' %s'E'>" % (
+ typename, mandatory_u_prefix)
+ assert str(cast(BWChar, 0x1234)) == "<cdata '%s' %s'\u1234'>" % (
+ typename, mandatory_u_prefix)
+ if not _hacked_pypy_uni4():
+ if wchar4:
x = cast(BWChar, 0x12345)
- assert str(x) == "<cdata 'wchar_t' %s'\U00012345'>" % (
- mandatory_u_prefix,)
+ assert str(x) == "<cdata '%s' %s'\U00012345'>" % (
+ typename, mandatory_u_prefix)
assert int(x) == 0x12345
- else:
- assert not pyuni4
+ else:
+ x = cast(BWChar, 0x18345)
+ assert str(x) == "<cdata '%s' %s'\u8345'>" % (
+ typename, mandatory_u_prefix)
+ assert int(x) == 0x8345
#
BWCharP = new_pointer_type(BWChar)
BStruct = new_struct_type("struct foo_s")
@@ -2117,9 +2143,9 @@
s.a1 = u+'\u1234'
assert s.a1 == u+'\u1234'
if pyuni4:
- assert wchar4
- s.a1 = u+'\U00012345'
- assert s.a1 == u+'\U00012345'
+ if wchar4:
+ s.a1 = u+'\U00012345'
+ assert s.a1 == u+'\U00012345'
elif wchar4:
if not _hacked_pypy_uni4():
s.a1 = cast(BWChar, 0x12345)
@@ -2154,17 +2180,17 @@
py.test.raises(IndexError, 'a[4]')
#
w = cast(BWChar, 'a')
- assert repr(w) == "<cdata 'wchar_t' %s'a'>" % mandatory_u_prefix
+ assert repr(w) == "<cdata '%s' %s'a'>" % (typename, mandatory_u_prefix)
assert str(w) == repr(w)
assert string(w) == u+'a'
assert int(w) == ord('a')
w = cast(BWChar, 0x1234)
- assert repr(w) == "<cdata 'wchar_t' %s'\u1234'>" % mandatory_u_prefix
+ assert repr(w) == "<cdata '%s' %s'\u1234'>" % (typename, mandatory_u_prefix)
assert str(w) == repr(w)
assert string(w) == u+'\u1234'
assert int(w) == 0x1234
w = cast(BWChar, u+'\u8234')
- assert repr(w) == "<cdata 'wchar_t' %s'\u8234'>" % mandatory_u_prefix
+ assert repr(w) == "<cdata '%s' %s'\u8234'>" % (typename, mandatory_u_prefix)
assert str(w) == repr(w)
assert string(w) == u+'\u8234'
assert int(w) == 0x8234
@@ -2172,8 +2198,8 @@
assert repr(w) == "<cdata 'int' 4660>"
if wchar4 and not _hacked_pypy_uni4():
w = cast(BWChar, u+'\U00012345')
- assert repr(w) == "<cdata 'wchar_t' %s'\U00012345'>" % (
- mandatory_u_prefix,)
+ assert repr(w) == "<cdata '%s' %s'\U00012345'>" % (
+ typename, mandatory_u_prefix)
assert str(w) == repr(w)
assert string(w) == u+'\U00012345'
assert int(w) == 0x12345
@@ -2200,7 +2226,7 @@
py.test.raises(RuntimeError, string, q)
#
def cb(p):
- assert repr(p).startswith("<cdata 'wchar_t *' 0x")
+ assert repr(p).startswith("<cdata '%s *' 0x" % typename)
return len(string(p))
BFunc = new_function_type((BWCharP,), BInt, False)
f = callback(BFunc, cb, -42)
@@ -2213,6 +2239,27 @@
x = cast(BWChar, -1)
py.test.raises(ValueError, string, x)
+def test_wchar_variants_mix():
+ BWChar = new_primitive_type("wchar_t")
+ BChar16 = new_primitive_type("char16_t")
+ BChar32 = new_primitive_type("char32_t")
+ assert int(cast(BChar32, cast(BChar16, -2))) == 0xfffe
+ assert int(cast(BWChar, cast(BChar16, -2))) == 0xfffe
+ assert int(cast(BChar16, cast(BChar32, 0x0001f345))) == 0xf345
+ assert int(cast(BChar16, cast(BWChar, 0x0001f345))) == 0xf345
+ #
+ BChar16A = new_array_type(new_pointer_type(BChar16), None)
+ BChar32A = new_array_type(new_pointer_type(BChar32), None)
+ x = cast(BChar32, 'A')
+ py.test.raises(TypeError, newp, BChar16A, [x])
+ x = cast(BChar16, 'A')
+ py.test.raises(TypeError, newp, BChar32A, [x])
+ #
+ a = newp(BChar16A, u+'\U00012345')
+ assert len(a) == 3
+ a = newp(BChar32A, u+'\U00012345')
+ assert len(a) == 2 # even if the Python unicode string above is 2 chars
+
def test_keepalive_struct():
# exception to the no-keepalive rule: p=newp(BStructPtr) returns a
# pointer owning the memory, and p[0] returns a pointer to the
@@ -3439,14 +3486,15 @@
py.test.raises(TypeError, "p[1:5] = u+'XYZT'")
py.test.raises(TypeError, "p[1:5] = [1, 2, 3, 4]")
#
- BUniChar = new_primitive_type("wchar_t")
- BArray = new_array_type(new_pointer_type(BUniChar), None)
- p = newp(BArray, u+"foobar")
- p[2:5] = [u+"*", u+"Z", u+"T"]
- p[1:3] = u+"XY"
- assert list(p) == [u+"f", u+"X", u+"Y", u+"Z", u+"T", u+"r", u+"\x00"]
- py.test.raises(TypeError, "p[1:5] = b'XYZT'")
- py.test.raises(TypeError, "p[1:5] = [1, 2, 3, 4]")
+ for typename in ["wchar_t", "char16_t", "char32_t"]:
+ BUniChar = new_primitive_type(typename)
+ BArray = new_array_type(new_pointer_type(BUniChar), None)
+ p = newp(BArray, u+"foobar")
+ p[2:5] = [u+"*", u+"Z", u+"T"]
+ p[1:3] = u+"XY"
+ assert list(p) == [u+"f", u+"X", u+"Y", u+"Z", u+"T", u+"r", u+"\x00"]
+ py.test.raises(TypeError, "p[1:5] = b'XYZT'")
+ py.test.raises(TypeError, "p[1:5] = [1, 2, 3, 4]")
def test_void_p_arithmetic():
BVoid = new_void_type()
@@ -3759,10 +3807,12 @@
p0 = p
assert unpack(p, 10) == b"abc\x00def\x00\x00\x00"
assert unpack(p+1, 5) == b"bc\x00de"
- BWChar = new_primitive_type("wchar_t")
- BArray = new_array_type(new_pointer_type(BWChar), 10) # wchar_t[10]
- p = newp(BArray, u"abc\x00def")
- assert unpack(p, 10) == u"abc\x00def\x00\x00\x00"
+
+ for typename in ["wchar_t", "char16_t", "char32_t"]:
+ BWChar = new_primitive_type(typename)
+ BArray = new_array_type(new_pointer_type(BWChar), 10) # wchar_t[10]
+ p = newp(BArray, u"abc\x00def")
+ assert unpack(p, 10) == u"abc\x00def\x00\x00\x00"
for typename, samples in [
("uint8_t", [0, 2**8-1]),
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -555,3 +555,11 @@
import _cffi_backend as _cffi1_backend
ffi = _cffi1_backend.FFI()
raises(ffi.error, ffi.cast, "int[-5]", 0)
+
+ def test_char32_t(self):
+ import _cffi_backend as _cffi1_backend
+ ffi = _cffi1_backend.FFI()
+ z = ffi.new("char32_t[]", u'\U00012345')
+ assert len(z) == 2
+ assert ffi.cast("int *", z)[0] == 0x12345
+ assert list(z) == [u'\U00012345', u'\x00'] # maybe a 2-unichars str
diff --git a/pypy/module/_cffi_backend/wchar_helper.py b/pypy/module/_cffi_backend/wchar_helper.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_cffi_backend/wchar_helper.py
@@ -0,0 +1,192 @@
+from rpython.rlib.objectmodel import specialize
+from rpython.rlib.rarithmetic import r_uint, r_ulonglong, intmask
+from rpython.rtyper.annlowlevel import llunicode
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.lltypesystem.rstr import copy_unicode_to_raw
+
+SIZE_UNICODE = rffi.sizeof(lltype.UniChar)
+
+
+if SIZE_UNICODE == 4:
+ def ordinal_to_unicode(ordinal): # 'ordinal' is a r_uint
+ return unichr(intmask(ordinal))
+else:
+ def ordinal_to_unicode(ordinal): # 'ordinal' is a r_uint
+ if ordinal <= 0xffff:
+ return unichr(intmask(ordinal))
+ elif ordinal <= 0x10ffff:
+ ordinal = intmask(ordinal - 0x10000)
+ return (unichr(0xD800 | (ordinal >> 10)) +
+ unichr(0xDC00 | (ordinal & 0x3FF)))
+ else:
+ raise OutOfRange(ordinal)
+
+def is_surrogate(u, index):
+ return (unichr(0xD800) <= u[index + 0] <= unichr(0xDBFF) and
+ unichr(0xDC00) <= u[index + 1] <= unichr(0xDFFF))
+
+def as_surrogate(u, index):
+ ordinal = (ord(u[index + 0]) - 0xD800) << 10
+ ordinal |= (ord(u[index + 1]) - 0xDC00)
+ return r_uint(ordinal + 0x10000)
+
+def unicode_to_ordinal(u):
+ if len(u) == 1:
+ u = ord(u[0])
+ return r_uint(u)
+ elif SIZE_UNICODE == 2:
+ if len(u) == 2 and is_surrogate(u, 0):
+ return r_uint(as_surrogate(u, 0))
+ raise ValueError
+
+
+class OutOfRange(Exception):
+ ordinal = 0
+
+ def __init__(self, ordinal):
+ ordinal = intmask(rffi.cast(rffi.INT, ordinal))
+ self.ordinal = ordinal
+
+def _unicode_from_wchar(ptr, length):
+ return rffi.wcharpsize2unicode(rffi.cast(rffi.CWCHARP, ptr), length)
+
+
+if SIZE_UNICODE == 2:
+ def unicode_from_char32(ptr, length):
+ # 'ptr' is a pointer to 'length' 32-bit integers
+ ptr = rffi.cast(rffi.UINTP, ptr)
+ alloc = length
+ for i in range(length):
+ if rffi.cast(lltype.Unsigned, ptr[i]) > 0xFFFF:
+ alloc += 1
+
+ u = [u'\x00'] * alloc
+ j = 0
+ for i in range(length):
+ ordinal = rffi.cast(lltype.Unsigned, ptr[i])
+ if ordinal > 0xFFFF:
+ if ordinal > 0x10FFFF:
+ raise OutOfRange(ordinal)
+ ordinal = intmask(ordinal - 0x10000)
+ u[j] = unichr(0xD800 | (ordinal >> 10))
+ j += 1
+ u[j] = unichr(0xDC00 | (ordinal & 0x3FF))
+ j += 1
+ else:
+ u[j] = unichr(intmask(ordinal))
+ j += 1
+ assert j == len(u)
+ return u''.join(u)
+
+ unicode_from_char16 = _unicode_from_wchar
+
+else:
+ unicode_from_char32 = _unicode_from_wchar
+
+ def unicode_from_char16(ptr, length):
+ # 'ptr' is a pointer to 'length' 16-bit integers
+ ptr = rffi.cast(rffi.USHORTP, ptr)
+ u = [u'\x00'] * length
+ i = 0
+ j = 0
+ while j < length:
+ ch = intmask(ptr[j])
+ j += 1
+ if 0xD800 <= ch <= 0xDBFF and j < length:
+ ch2 = intmask(ptr[j])
+ if 0xDC00 <= ch2 <= 0xDFFF:
+ ch = (((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000
+ j += 1
+ u[i] = unichr(ch)
+ i += 1
+ del u[i:]
+ return u''.join(u)
+
+
+ at specialize.ll()
+def _measure_length(ptr, maxlen):
+ result = 0
+ if maxlen < 0:
+ while intmask(ptr[result]) != 0:
+ result += 1
+ else:
+ while result < maxlen and intmask(ptr[result]) != 0:
+ result += 1
+ return result
+
+def measure_length_16(ptr, maxlen=-1):
+ return _measure_length(rffi.cast(rffi.USHORTP, ptr), maxlen)
+
+def measure_length_32(ptr, maxlen=-1):
+ return _measure_length(rffi.cast(rffi.UINTP, ptr), maxlen)
+
+
+def unicode_size_as_char16(u):
+ result = len(u)
+ if SIZE_UNICODE == 4:
+ for i in range(result):
+ if ord(u[i]) > 0xFFFF:
+ result += 1
+ return result
+
+def unicode_size_as_char32(u):
+ result = len(u)
+ if SIZE_UNICODE == 2 and result > 1:
+ for i in range(result - 1):
+ if is_surrogate(u, i):
+ result -= 1
+ return result
+
+
+def _unicode_to_wchar(u, target_ptr, target_length, add_final_zero):
+ # 'target_ptr' is a raw pointer to 'target_length' wchars;
+ # we assume here that target_length == len(u).
+ unichardata = rffi.cast(rffi.CWCHARP, target_ptr)
+ copy_unicode_to_raw(llunicode(u), unichardata, 0, target_length)
+ if add_final_zero:
+ unichardata[target_length] = u'\x00'
+
+
+if SIZE_UNICODE == 2:
+ def unicode_to_char32(u, target_ptr, target_length, add_final_zero):
+ # 'target_ptr' is a raw pointer to 'target_length' 32-bit integers;
+ # we assume here that target_length == unicode_size_as_char32(u).
+ ptr = rffi.cast(rffi.UINTP, target_ptr)
+ src_index = 0
+ last_surrogate_pos = len(u) - 2
+ for i in range(target_length):
+ if src_index <= last_surrogate_pos and is_surrogate(u, src_index):
+ ordinal = as_surrogate(u, src_index)
+ src_index += 2
+ else:
+ ordinal = r_uint(ord(u[src_index]))
+ src_index += 1
+ ptr[i] = rffi.cast(rffi.UINT, ordinal)
+ if add_final_zero:
+ ptr[target_length] = rffi.cast(rffi.UINT, 0)
+
+ unicode_to_char16 = _unicode_to_wchar
+
+else:
+ unicode_to_char32 = _unicode_to_wchar
+
+ def unicode_to_char16(u, target_ptr, target_length, add_final_zero):
+ # 'target_ptr' is a raw pointer to 'target_length' 16-bit integers;
+ # we assume here that target_length == unicode_size_as_char16(u).
+ ptr = rffi.cast(rffi.USHORTP, target_ptr)
+ for uc in u:
+ ordinal = ord(uc)
+ if ordinal > 0xFFFF:
+ if ordinal > 0x10FFFF:
+ raise OutOfRange(ordinal)
+ ordinal -= 0x10000
+ ptr[0] = rffi.cast(rffi.USHORT, 0xD800 | (ordinal >> 10))
+ ptr[1] = rffi.cast(rffi.USHORT, 0xDC00 | (ordinal & 0x3FF))
+ ptr = rffi.ptradd(ptr, 2)
+ else:
+ ptr[0] = rffi.cast(rffi.USHORT, ordinal)
+ ptr = rffi.ptradd(ptr, 1)
+ assert ptr == (
+ rffi.ptradd(rffi.cast(rffi.USHORTP, target_ptr), target_length))
+ if add_final_zero:
+ ptr[0] = rffi.cast(rffi.USHORT, 0)
diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/__init__.py
--- a/pypy/module/_vmprof/__init__.py
+++ b/pypy/module/_vmprof/__init__.py
@@ -14,6 +14,9 @@
'write_all_code_objects': 'interp_vmprof.write_all_code_objects',
'is_enabled': 'interp_vmprof.is_enabled',
'get_profile_path': 'interp_vmprof.get_profile_path',
+ 'stop_sampling': 'interp_vmprof.stop_sampling',
+ 'start_sampling': 'interp_vmprof.start_sampling',
+
'VMProfError': 'space.fromcache(interp_vmprof.Cache).w_VMProfError',
}
diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -51,8 +51,8 @@
return OperationError(w_VMProfError, space.newtext(e.msg))
- at unwrap_spec(fileno=int, period=float, memory=int, lines=int, native=int)
-def enable(space, fileno, period, memory, lines, native):
+ at unwrap_spec(fileno=int, period=float, memory=int, lines=int, native=int, real_time=int)
+def enable(space, fileno, period, memory, lines, native, real_time):
"""Enable vmprof. Writes go to the given 'fileno', a file descriptor
opened for writing. *The file descriptor must remain open at least
until disable() is called.*
@@ -66,7 +66,7 @@
# "with vmprof will crash"),
# space.w_RuntimeWarning)
try:
- rvmprof.enable(fileno, period, memory, native)
+ rvmprof.enable(fileno, period, memory, native, real_time)
except rvmprof.VMProfError as e:
raise VMProfError(space, e)
@@ -96,3 +96,10 @@
More information about the pypy-commit
mailing list