From pypy.commits at gmail.com  Sat Aug  4 00:10:01 2018
From: pypy.commits at gmail.com (wlav)
Date: Fri, 03 Aug 2018 21:10:01 -0700 (PDT)
Subject: [pypy-commit] pypy default: std::vector performance and handle long
 double through narrowing
Message-ID: <5b652719.1c69fb81.f853d.f09a@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: 
Changeset: r94934:cc09dd3d75c4
Date: 2018-08-03 17:44 -0700
http://bitbucket.org/pypy/pypy/changeset/cc09dd3d75c4/

Log:	std::vector performance and handle long double through narrowing

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -8,7 +8,7 @@
 .. branch: cppyy-packaging
 
 Main items: vastly better template resolution and improved performance. In
-detail: upgrade to backend 1.3.1, improved handling of templated methods and
+detail: upgrade to backend 1.4, improved handling of templated methods and
 functions (in particular automatic deduction of types), improved pythonization
 interface, range of compatibility fixes for Python3, free functions now take
 fast libffi path when possible, moves for strings (incl. from Python str),
diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py
--- a/pypy/module/_cppyy/capi/loadable_capi.py
+++ b/pypy/module/_cppyy/capi/loadable_capi.py
@@ -1,13 +1,18 @@
 import os
+
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib import jit, jit_libffi, libffi, rdynload, objectmodel
 from rpython.rlib.rarithmetic import r_singlefloat
 from rpython.tool import leakfinder
 
-from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.error import oefmt
+from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.argument import Arguments
+from pypy.interpreter.gateway import interp2app, interpindirect2app
+from pypy.interpreter.typedef import TypeDef
+from pypy.objspace.std.iterobject import W_AbstractSeqIterObject
 
+from pypy.module._rawffi.array import W_ArrayInstance
 from pypy.module._cffi_backend import ctypefunc, ctypeprim, cdataobj, misc
 from pypy.module._cffi_backend import newtype
 from pypy.module._cppyy import ffitypes
@@ -23,10 +28,11 @@
 
 class _Arg:         # poor man's union
     _immutable_ = True
-    def __init__(self, tc, h = 0, l = -1, s = '', p = rffi.cast(rffi.VOIDP, 0)):
+    def __init__(self, tc, h = 0, l = -1, d = -1., s = '', p = rffi.cast(rffi.VOIDP, 0)):
         self.tc      = tc
         self._handle = h
         self._long   = l
+        self._double = d
         self._string = s
         self._voidp  = p
 
@@ -40,6 +46,11 @@
     def __init__(self, val):
         _Arg.__init__(self, 'l', l = val)
 
+class _ArgD(_Arg):
+    _immutable_ = True
+    def __init__(self, val):
+        _Arg.__init__(self, 'd', d = val)
+
 class _ArgS(_Arg):
     _immutable_ = True
     def __init__(self, val):
@@ -89,6 +100,9 @@
                     assert obj._voidp != rffi.cast(rffi.VOIDP, 0)
                     data = rffi.cast(rffi.VOIDPP, data)
                     data[0] = obj._voidp
+                elif obj.tc == 'd':
+                    assert isinstance(argtype, ctypeprim.W_CTypePrimitiveFloat)
+                    misc.write_raw_float_data(data, rffi.cast(rffi.DOUBLE, obj._double), argtype.size)
                 else:    # only other use is string
                     assert obj.tc == 's'
                     n = len(obj._string)
@@ -182,6 +196,7 @@
             'call_f'       : ([c_method, c_object, c_int, c_voidp],   c_float),
             'call_d'       : ([c_method, c_object, c_int, c_voidp],   c_double),
             'call_ld'      : ([c_method, c_object, c_int, c_voidp],   c_ldouble),
+            'call_nld'     : ([c_method, c_object, c_int, c_voidp],   c_double),
 
             'call_r'       : ([c_method, c_object, c_int, c_voidp],   c_voidp),
             # call_s actually takes an size_t* as last parameter, but this will do
@@ -274,8 +289,9 @@
             'stdstring2charp'          : ([c_object, c_voidp],        c_ccharp),
             'stdstring2stdstring'      : ([c_object],                 c_object),
 
-            'stdvector_valuetype'      : ([c_ccharp],                 c_ccharp),
-            'stdvector_valuesize'      : ([c_ccharp],                 c_size_t),
+            'longdouble2double'        : ([c_voidp],                  c_double),
+            'double2longdouble'        : ([c_double, c_voidp],        c_void),
+
             'vectorbool_getitem'       : ([c_object, c_int],          c_int),
             'vectorbool_setitem'       : ([c_object, c_int, c_int],   c_void),
         }
@@ -404,7 +420,9 @@
     return rffi.cast(rffi.DOUBLE, space.float_w(call_capi(space, 'call_d', args)))
 def c_call_ld(space, cppmethod, cppobject, nargs, cargs):
     args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs)]
-    return rffi.cast(rffi.LONGDOUBLE, space.float_w(call_capi(space, 'call_ld', args)))
+    #return rffi.cast(rffi.LONGDOUBLE, space.float_w(call_capi(space, 'call_ld', args)))
+    # call_nld narrows long double to double
+    return rffi.cast(rffi.DOUBLE, space.float_w(call_capi(space, 'call_nld', args)))
 
 def c_call_r(space, cppmethod, cppobject, nargs, cargs):
     args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs)]
@@ -658,13 +676,11 @@
 def c_stdstring2stdstring(space, cppobject):
     return _cdata_to_cobject(space, call_capi(space, 'stdstring2stdstring', [_ArgH(cppobject)]))
 
-def c_stdvector_valuetype(space, pystr):
-    return charp2str_free(space, call_capi(space, 'stdvector_valuetype', [_ArgS(pystr)]))
+def c_longdouble2double(space, addr):
+    return space.float_w(call_capi(space, 'longdouble2double', [_ArgP(addr)]))
+def c_double2longdouble(space, dval, addr):
+    call_capi(space, 'double2longdouble', [_ArgD(dval), _ArgP(addr)])
 
-def c_stdvector_valuetype(space, pystr):
-    return charp2str_free(space, call_capi(space, 'stdvector_valuetype', [_ArgS(pystr)]))
-def c_stdvector_valuesize(space, pystr):
-    return _cdata_to_size_t(space, call_capi(space, 'stdvector_valuesize', [_ArgS(pystr)]))
 def c_vectorbool_getitem(space, vbool, idx):
     return call_capi(space, 'vectorbool_getitem', [_ArgH(vbool), _ArgL(idx)])
 def c_vectorbool_setitem(space, vbool, idx, value):
@@ -702,6 +718,52 @@
     idx = vbool_getindex(space, w_self, w_idx)
     c_vectorbool_setitem(space, vbool._rawobject, idx, int(space.is_true(w_value)))
 
+class W_STLVectorIter(W_AbstractSeqIterObject):
+    # w_seq and index are in base class
+    _immutable_fields_ = ['converter', 'data', 'len', 'stride']
+
+    def __init__(self, space, w_vector):
+        W_AbstractSeqIterObject.__init__(self, w_vector)
+        # TODO: this should live in rpythonize.py or something so that the
+        # imports can move to the top w/o getting circles
+        from pypy.module._cppyy import interp_cppyy
+        assert isinstance(w_vector, interp_cppyy.W_CPPInstance)
+        vector = space.interp_w(interp_cppyy.W_CPPInstance, w_vector)
+
+        v_type = c_resolve_name(space, vector.clsdecl.name+'::value_type')
+        v_size = c_size_of_type(space, v_type)
+
+        if not v_type or not v_size:
+            raise NotImplementedError   # fallback on getitem
+
+        from pypy.module._cppyy import converter
+        self.converter = converter.get_converter(space, v_type, '')
+
+        # this 'data' is from the decl, so not the pythonized data from pythonify.py
+        w_arr = space.call_obj_args(vector.clsdecl.get_overload('data'), w_vector, Arguments(space, []))
+        arr = space.interp_w(W_ArrayInstance, w_arr, can_be_None=True)
+        if not arr:
+            raise OperationError(space.w_StopIteration, space.w_None)
+
+        self.data    = rffi.cast(rffi.CCHARP, space.uint_w(arr.getbuffer(space)))
+        self.len     = space.uint_w(space.call_obj_args(vector.clsdecl.get_overload('size'), w_vector, Arguments(space, [])))
+        self.stride  = v_size
+
+    def descr_next(self, space):
+        if self.w_seq is None:
+            raise OperationError(space.w_StopIteration, space.w_None)
+        if self.len <= self.index:
+            self.w_seq = None
+            raise OperationError(space.w_StopIteration, space.w_None)
+        offset = lltype.direct_ptradd(self.data, rffi.cast(rffi.SIZE_T, self.index*self.stride))
+        w_item = self.converter.from_memory(space, space.w_None, rffi.cast(rffi.LONG, offset))
+        self.index += 1
+        return w_item
+
+def stdvector_iter(space, w_self):
+    return W_STLVectorIter(space, w_self)
+
+
 # setup pythonizations for later use at run-time
 _pythonizations = {}
 def register_pythonizations(space):
@@ -712,6 +774,9 @@
         ### std::string
         stdstring_c_str,
 
+        ### std::vector
+        stdvector_iter,
+
         ### std::vector<bool>
         vectorbool_getitem,
         vectorbool_setitem,
@@ -730,6 +795,9 @@
         _method_alias(space, w_pycppclass, "_cppyy_as_builtin", "c_str")
         _method_alias(space, w_pycppclass, "__str__",           "c_str")
 
-    if name == "std::vector<bool>":
+    if  name.find("std::vector<bool", 0, 16) == 0:
         space.setattr(w_pycppclass, space.newtext("__getitem__"), _pythonizations["vectorbool_getitem"])
         space.setattr(w_pycppclass, space.newtext("__setitem__"), _pythonizations["vectorbool_setitem"])
+
+    elif name.find("std::vector", 0, 11) == 0:
+        space.setattr(w_pycppclass, space.newtext("__iter__"), _pythonizations["stdvector_iter"])
diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -211,6 +211,9 @@
         x[0] = self._unwrap_object(space, w_obj)
 
     def default_argument_libffi(self, space, address):
+        if not self.valid_default:
+            from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
+            raise FastCallNotPossible
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self.default
 
@@ -224,7 +227,7 @@
         rffiptr = rffi.cast(self.c_ptrtype, address)
         rffiptr[0] = self._unwrap_object(space, w_value)
 
-class ConstRefNumericTypeConverterMixin(NumericTypeConverterMixin):
+class ConstRefNumericTypeConverterMixin(object):
     _mixin_ = True
 
     def cffi_type(self, space):
@@ -321,59 +324,92 @@
     pass
 
 class FloatConverter(ffitypes.typeid(rffi.FLOAT), FloatTypeConverterMixin, TypeConverter):
-    _immutable_fields_ = ['default']
+    _immutable_fields_ = ['default', 'valid_default']
 
     def __init__(self, space, default):
-        if default:
+        self.valid_default = False
+        try:
             fval = float(rfloat.rstring_to_float(default))
-        else:
+            self.valid_default = True
+        except Exception:
             fval = float(0.)
-        self.default = r_singlefloat(fval)
+        self.default = rffi.cast(rffi.FLOAT, r_singlefloat(fval))
 
     def from_memory(self, space, w_obj, offset):
         address = self._get_raw_address(space, w_obj, offset)
         rffiptr = rffi.cast(self.c_ptrtype, address)
         return self._wrap_object(space, rffiptr[0])
 
-class ConstFloatRefConverter(FloatConverter):
+class ConstFloatRefConverter(ConstRefNumericTypeConverterMixin, FloatConverter):
     _immutable_fields_ = ['typecode']
     typecode = 'f'
 
-    def cffi_type(self, space):
-        state = space.fromcache(ffitypes.State)
-        return state.c_voidp
-
-    def convert_argument_libffi(self, space, w_obj, address, scratch):
-        from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
-        raise FastCallNotPossible
-
 class DoubleConverter(ffitypes.typeid(rffi.DOUBLE), FloatTypeConverterMixin, TypeConverter):
-    _immutable_fields_ = ['default']
+    _immutable_fields_ = ['default', 'valid_default']
 
     def __init__(self, space, default):
-        if default:
+        self.valid_default = False
+        try:
             self.default = rffi.cast(self.c_type, rfloat.rstring_to_float(default))
-        else:
+            self.valid_default = True
+        except Exception:
             self.default = rffi.cast(self.c_type, 0.)
 
 class ConstDoubleRefConverter(ConstRefNumericTypeConverterMixin, DoubleConverter):
     _immutable_fields_ = ['typecode']
     typecode = 'd'
 
-class LongDoubleConverter(ffitypes.typeid(rffi.LONGDOUBLE), FloatTypeConverterMixin, TypeConverter):
-    _immutable_fields_ = ['default']
+class LongDoubleConverter(TypeConverter):
+    _immutable_fields_ = ['default', 'valid_default']
+    typecode = 'g'
 
     def __init__(self, space, default):
-        if default:
-            fval = float(rfloat.rstring_to_float(default))
-        else:
-            fval = float(0.)
-        self.default = r_longfloat(fval)
+        self.valid_default = False
+        try:
+            # use float() instead of cast with r_longfloat
+            fval = rffi.cast(rffi.DOUBLE, rfloat.rstring_to_float(default))
+            self.valid_default = True
+        except Exception:
+            fval = rffi.cast(rffi.DOUBLE, 0.)
+        #self.default = r_longfloat(fval)
+        self.default = fval
+
+    def convert_argument(self, space, w_obj, address):
+        x = rffi.cast(rffi.VOIDP, address)
+        capi.c_double2longdouble(space, space.float_w(w_obj), x)
+        ba = rffi.cast(rffi.CCHARP, address)
+        ba[capi.c_function_arg_typeoffset(space)] = self.typecode
+
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
+        x = rffi.cast(rffi.VOIDP, address)
+        capi.c_double2longdouble(space, space.float_w(w_obj), x)
+
+    def default_argument_libffi(self, space, address):
+        if not self.valid_default:
+            from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
+            raise FastCallNotPossible
+        x = rffi.cast(rffi.VOIDP, address)
+        capi.c_double2longdouble(space, self.default, x)
+
+    def from_memory(self, space, w_obj, offset):
+        address = self._get_raw_address(space, w_obj, offset)
+        rffiptr = rffi.cast(rffi.VOIDP, address)
+        return space.newfloat(capi.c_longdouble2double(space, rffiptr))
+
+    def to_memory(self, space, w_obj, w_value, offset):
+        address = self._get_raw_address(space, w_obj, offset)
+        rffiptr = rffi.cast(rffi.VOIDP, address)
+        capi.c_double2longdouble(space, space.float_w(w_value), rffiptr)
 
 class ConstLongDoubleRefConverter(ConstRefNumericTypeConverterMixin, LongDoubleConverter):
     _immutable_fields_ = ['typecode']
     typecode = 'g'
 
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
+        capi.c_double2longdouble(space, space.float_w(w_obj), rffi.cast(rffi.VOIDP, scratch))
+        x = rffi.cast(rffi.VOIDPP, address)
+        x[0] = scratch
+
 
 class CStringConverter(TypeConverter):
     def convert_argument(self, space, w_obj, address):
@@ -949,8 +985,8 @@
 _converters["const float&"]             = ConstFloatRefConverter
 _converters["double"]                   = DoubleConverter
 _converters["const double&"]            = ConstDoubleRefConverter
-#_converters["long double"]              = LongDoubleConverter
-#_converters["const long double&"]       = ConstLongDoubleRefConverter
+_converters["long double"]              = LongDoubleConverter
+_converters["const long double&"]       = ConstLongDoubleRefConverter
 _converters["const char*"]              = CStringConverter
 _converters["void*"]                    = VoidPtrConverter
 _converters["void**"]                   = VoidPtrPtrConverter
@@ -984,7 +1020,12 @@
             _immutable_ = True
             typecode = c_tc
             def __init__(self, space, default):
-                self.default = rffi.cast(self.c_type, capi.c_strtoll(space, default))
+                self.valid_default = False
+                try:
+                    self.default = rffi.cast(self.c_type, capi.c_strtoll(space, default))
+                    self.valid_default = True
+                except Exception:
+                    self.default = rffi.cast(self.c_type, 0)
         class ConstRefConverter(ConstRefNumericTypeConverterMixin, BasicConverter):
             _immutable_ = True
         for name in names:
@@ -1001,7 +1042,12 @@
             _immutable_ = True
             typecode = c_tc
             def __init__(self, space, default):
-                self.default = rffi.cast(self.c_type, capi.c_strtoll(space, default))
+                self.valid_default = False
+                try:
+                    self.default = rffi.cast(self.c_type, capi.c_strtoll(space, default))
+                    self.valid_default = True
+                except Exception:
+                    self.default = rffi.cast(self.c_type, 0)
         class ConstRefConverter(ConstRefNumericTypeConverterMixin, BasicConverter):
             _immutable_ = True
         for name in names:
@@ -1021,7 +1067,12 @@
             _immutable_ = True
             typecode = c_tc
             def __init__(self, space, default):
-                self.default = rffi.cast(self.c_type, capi.c_strtoull(space, default))
+                self.valid_default = False
+                try:
+                    self.default = rffi.cast(self.c_type, capi.c_strtoull(space, default))
+                    self.valid_default = True
+                except Exception:
+                    self.default = rffi.cast(self.c_type, 0)
         class ConstRefConverter(ConstRefNumericTypeConverterMixin, BasicConverter):
             _immutable_ = True
         for name in names:
diff --git a/pypy/module/_cppyy/executor.py b/pypy/module/_cppyy/executor.py
--- a/pypy/module/_cppyy/executor.py
+++ b/pypy/module/_cppyy/executor.py
@@ -76,9 +76,6 @@
 class NumericExecutorMixin(object):
     _mixin_ = True
 
-    #def _wrap_object(self, space, obj):
-    #    return getattr(space, self.wrapper)(obj)
-
     def execute(self, space, cppmethod, cppthis, num_args, args):
         result = self.c_stubcall(space, cppmethod, cppthis, num_args, args)
         return self._wrap_object(space, rffi.cast(self.c_type, result))
@@ -100,13 +97,10 @@
         self.w_item = w_item
         self.do_assign = True
 
-    #def _wrap_object(self, space, obj):
-    #    return getattr(space, self.wrapper)(rffi.cast(self.c_type, obj))
-
     def _wrap_reference(self, space, rffiptr):
         if self.do_assign:
             rffiptr[0] = rffi.cast(self.c_type, self._unwrap_object(space, self.w_item))
-        self.do_assign = False
+            self.do_assign = False
         return self._wrap_object(space, rffiptr[0])    # all paths, for rtyper
 
     def execute(self, space, cppmethod, cppthis, num_args, args):
@@ -119,6 +113,48 @@
         return self._wrap_reference(space,
             rffi.cast(self.c_ptrtype, rffi.cast(rffi.VOIDPP, result)[0]))
 
+class LongDoubleExecutorMixin(object):
+    # Note: not really supported, but returns normal double
+    _mixin_ = True
+
+    def execute(self, space, cppmethod, cppthis, num_args, args):
+        result = self.c_stubcall(space, cppmethod, cppthis, num_args, args)
+        return space.newfloat(result)
+
+    def execute_libffi(self, space, cif_descr, funcaddr, buffer):
+        from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
+        raise FastCallNotPossible
+
+class LongDoubleExecutor(ffitypes.typeid(rffi.LONGDOUBLE), LongDoubleExecutorMixin, Executor):
+    _immutable_ = True
+    c_stubcall  = staticmethod(capi.c_call_ld)
+
+class LongDoubleRefExecutorMixin(NumericRefExecutorMixin):
+    # Note: not really supported, but returns normal double
+    _mixin_ = True
+
+    def _wrap_reference(self, space, rffiptr):
+        if self.do_assign:
+            capi.c_double2longdouble(space, space.float_w(self.w_item), rffiptr)
+            self.do_assign = False
+            return self.w_item
+        return space.newfloat(capi.c_longdouble2double(space, rffiptr))
+
+    def execute(self, space, cppmethod, cppthis, num_args, args):
+        result = capi.c_call_r(space, cppmethod, cppthis, num_args, args)
+        return self._wrap_reference(space, rffi.cast(self.c_ptrtype, result))
+
+    def execute_libffi(self, space, cif_descr, funcaddr, buffer):
+        jit_libffi.jit_ffi_call(cif_descr, funcaddr, buffer)
+        result = rffi.ptradd(buffer, cif_descr.exchange_result)
+        return self._wrap_reference(space,
+            rffi.cast(self.c_ptrtype, rffi.cast(rffi.VOIDPP, result)[0]))
+
+class LongDoubleRefExecutor(ffitypes.typeid(rffi.LONGDOUBLE), LongDoubleRefExecutorMixin, Executor):
+    def cffi_type(self, space):
+        state = space.fromcache(ffitypes.State)
+        return state.c_voidp
+
 
 class CStringExecutor(Executor):
     def execute(self, space, cppmethod, cppthis, num_args, args):
@@ -341,6 +377,10 @@
 _executors["void*"]               = PtrTypeExecutor
 _executors["const char*"]         = CStringExecutor
 
+# long double not really supported: narrows to double
+_executors["long double"]          = LongDoubleExecutor
+_executors["long double&"]         = LongDoubleRefExecutor
+
 # special cases (note: 'string' aliases added below)
 _executors["constructor"]         = ConstructorExecutor
 
diff --git a/pypy/module/_cppyy/ffitypes.py b/pypy/module/_cppyy/ffitypes.py
--- a/pypy/module/_cppyy/ffitypes.py
+++ b/pypy/module/_cppyy/ffitypes.py
@@ -296,7 +296,8 @@
     _immutable_fields_ = ['c_type', 'c_ptrtype', 'typecode']
 
     c_type      = rffi.LONGDOUBLE
-    c_ptrtype   = rffi.LONGDOUBLEP
+    # c_ptrtype   = rffi.LONGDOUBLEP   # useless type at this point
+    c_ptrtype   = rffi.VOIDP
     typecode    = 'g'
 
     # long double is not really supported ...
@@ -304,7 +305,7 @@
         return r_longfloat(space.float_w(w_obj))
 
     def _wrap_object(self, space, obj):
-        return space.wrap(obj)
+        return space.newfloat(obj)
 
     def cffi_type(self, space):
         state = space.fromcache(State)
diff --git a/pypy/module/_cppyy/include/capi.h b/pypy/module/_cppyy/include/capi.h
--- a/pypy/module/_cppyy/include/capi.h
+++ b/pypy/module/_cppyy/include/capi.h
@@ -63,6 +63,8 @@
     double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
     RPY_EXTERN
     long double cppyy_call_ld(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
+    double cppyy_call_nld(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
 
     RPY_EXTERN
     void* cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
@@ -220,9 +222,10 @@
     cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr);
 
     RPY_EXTERN
-    const char* cppyy_stdvector_valuetype(const char* clname);
+    double cppyy_longdouble2double(void*);
     RPY_EXTERN
-    size_t      cppyy_stdvector_valuesize(const char* clname);
+    void   cppyy_double2longdouble(double, void*);
+
     RPY_EXTERN
     int         cppyy_vectorbool_getitem(cppyy_object_t ptr, int idx);
     RPY_EXTERN
diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -230,8 +230,10 @@
         if self.converters is None:
             try:
                 self._setup(cppthis)
-            except Exception:
-                pass
+            except Exception as e:
+                if self.converters is None:
+                    raise oefmt(self.space.w_SystemError,
+                        "unable to initialize converters (%s)", str(e))
 
         # attempt to call directly through ffi chain
         if useffi and self._funcaddr:
@@ -848,7 +850,7 @@
                 method = self.master.overloads[name]
             else:
             # try to match with run-time instantiations
-                # TODO: logically, this could be used, but in practice, it's proving to
+                # TODO: logically, this could be used, but in practice, it's proving too
                 #  greedy ... maybe as a last resort?
                 #for cppol in self.master.overloads.values():
                 #    try:
@@ -1221,6 +1223,12 @@
         self.datamembers[name] = new_dm
         return new_dm
 
+    @unwrap_spec(name='text')
+    def has_enum(self, name):
+        if capi.c_is_enum(self.space, self.name+'::'+name):
+            return self.space.w_True
+        return self.space.w_False
+
     def _encode_dm_dimensions(self, idata):
         # encode dimensions (TODO: this is ugly, but here's where the info is)
         dims = []
@@ -1269,6 +1277,8 @@
 
     def _make_datamember(self, dm_name, dm_idx):
         type_name = capi.c_datamember_type(self.space, self, dm_idx)
+        if capi.c_is_enum_data(self.space, self, dm_idx):
+            type_name = capi.c_resolve_enum(self.space, type_name)
         offset = capi.c_datamember_offset(self.space, self, dm_idx)
         if offset == -1:
             raise self.missing_attribute_error(dm_name)
@@ -1326,6 +1336,7 @@
     get_datamember_names = interp2app(W_CPPNamespaceDecl.get_datamember_names),
     get_datamember = interp2app(W_CPPNamespaceDecl.get_datamember),
     is_namespace = interp2app(W_CPPNamespaceDecl.is_namespace),
+    has_enum = interp2app(W_CPPNamespaceDecl.has_enum),
     __cppname__ = interp_attrproperty('name', W_CPPNamespaceDecl, wrapfn="newtext"),
     __dispatch__ = interp2app(W_CPPNamespaceDecl.scope__dispatch__),
     __dir__ = interp2app(W_CPPNamespaceDecl.ns__dir__),
@@ -1482,6 +1493,7 @@
     get_datamember_names = interp2app(W_CPPClassDecl.get_datamember_names),
     get_datamember = interp2app(W_CPPClassDecl.get_datamember),
     is_namespace = interp2app(W_CPPClassDecl.is_namespace),
+    has_enum = interp2app(W_CPPClassDecl.has_enum),
     __cppname__ = interp_attrproperty('name', W_CPPClassDecl, wrapfn="newtext"),
     __dispatch__ = interp2app(W_CPPClassDecl.scope__dispatch__)
 )
diff --git a/pypy/module/_cppyy/lowlevelviews.py b/pypy/module/_cppyy/lowlevelviews.py
--- a/pypy/module/_cppyy/lowlevelviews.py
+++ b/pypy/module/_cppyy/lowlevelviews.py
@@ -57,7 +57,7 @@
     itemaddress = interp2app(W_LowLevelView.descr_itemaddress),
     reshape     = interp2app(W_LowLevelView.reshape),
 )
-W_ArrayInstance.typedef.acceptable_as_base_class = False
+W_LowLevelView.typedef.acceptable_as_base_class = False
 
 
 class W_ArrayOfInstances(W_Root):
diff --git a/pypy/module/_cppyy/pythonify.py b/pypy/module/_cppyy/pythonify.py
--- a/pypy/module/_cppyy/pythonify.py
+++ b/pypy/module/_cppyy/pythonify.py
@@ -333,6 +333,11 @@
         except AttributeError:
             pass
 
+    # enum type
+    if not cppitem:
+        if scope.__cppdecl__.has_enum(name):
+            pycppitem = int
+
     if pycppitem is not None:      # pycppitem could be a bound C++ NULL, so check explicitly for Py_None
         return pycppitem
 
@@ -361,18 +366,19 @@
     return '', name
 
 # pythonization by decoration (move to their own file?)
-def python_style_getitem(self, idx):
+def python_style_getitem(self, _idx):
     # python-style indexing: check for size and allow indexing from the back
-    try:
-        sz = len(self)
+    sz = len(self)
+    idx = _idx
+    if isinstance(idx, int):
         if idx < 0: idx = sz + idx
-        if idx < sz:
+        if 0 <= idx < sz:
             return self._getitem__unchecked(idx)
-        raise IndexError(
-            'index out of range: %d requested for %s of size %d' % (idx, str(self), sz))
-    except TypeError:
-        pass
-    return self._getitem__unchecked(idx)
+        else:
+            raise IndexError(
+                'index out of range: %s requested for %s of size %d' % (str(idx), str(self), sz))
+    # may fail for the same reasons as above, but will now give proper error message
+    return self._getitem__unchecked(_idx)
 
 def python_style_sliceable_getitem(self, slice_or_idx):
     if type(slice_or_idx) == slice:
@@ -380,8 +386,7 @@
         nseq += [python_style_getitem(self, i) \
                     for i in range(*slice_or_idx.indices(len(self)))]
         return nseq
-    else:
-        return python_style_getitem(self, slice_or_idx)
+    return python_style_getitem(self, slice_or_idx)
 
 def _pythonize(pyclass, name):
     # general note: use 'in pyclass.__dict__' rather than 'hasattr' to prevent
@@ -430,8 +435,8 @@
     # map begin()/end() protocol to iter protocol on STL(-like) classes, but
     # not on vector, which is pythonized in the capi (interp-level; there is
     # also the fallback on the indexed __getitem__, but that is slower)
-# TODO:    if not (0 <= name.find('vector') <= 5):
-    if not (0 <= name.find('vector<bool') <= 5):
+    add_checked_item = False
+    if name.find('std::vector', 0, 11) != 0:
         if ('begin' in pyclass.__dict__ and 'end' in pyclass.__dict__):
             if _cppyy._scope_byname(name+'::iterator') or \
                     _cppyy._scope_byname(name+'::const_iterator'):
@@ -443,10 +448,12 @@
                     i.__destruct__()
                     raise StopIteration
                 pyclass.__iter__ = __iter__
-            # else: rely on numbered iteration
+            else:
+                # rely on numbered iteration
+                add_checked_item = True
 
     # add python collection based initializer
-    if 0 <= name.find('vector') <= 5:
+    if name.find('std::vector', 0, 11) == 0:
         pyclass.__real_init__ = pyclass.__init__
         def vector_init(self, *args):
             if len(args) == 1 and isinstance(args[0], (tuple, list)):
@@ -468,13 +475,17 @@
                 return arr
             pyclass.data = data_with_len
 
-    # combine __getitem__ and __len__ to make a pythonized __getitem__
-    if '__getitem__' in pyclass.__dict__ and '__len__' in pyclass.__dict__:
-        pyclass._getitem__unchecked = pyclass.__getitem__
-        if '__setitem__' in pyclass.__dict__ and '__iadd__' in pyclass.__dict__:
-            pyclass.__getitem__ = python_style_sliceable_getitem
-        else:
-            pyclass.__getitem__ = python_style_getitem
+    # TODO: must be a simpler way to check (or at least hook these to a namespace
+    # std specific pythonizor)
+    if add_checked_item or name.find('std::vector', 0, 11) == 0 or \
+            name.find('std::array', 0, 11) == 0 or name.find('std::deque', 0, 10) == 0:
+        # combine __getitem__ and __len__ to make a pythonized __getitem__
+        if '__getitem__' in pyclass.__dict__ and '__len__' in pyclass.__dict__:
+            pyclass._getitem__unchecked = pyclass.__getitem__
+            if '__setitem__' in pyclass.__dict__ and '__iadd__' in pyclass.__dict__:
+                pyclass.__getitem__ = python_style_sliceable_getitem
+            else:
+                pyclass.__getitem__ = python_style_getitem
 
     # string comparisons
     if name == _cppyy._std_string_name():
diff --git a/pypy/module/_cppyy/src/dummy_backend.cxx b/pypy/module/_cppyy/src/dummy_backend.cxx
--- a/pypy/module/_cppyy/src/dummy_backend.cxx
+++ b/pypy/module/_cppyy/src/dummy_backend.cxx
@@ -62,6 +62,7 @@
         m_name(name), m_argtypes(argtypes), m_returntype(returntype), m_type(mtype) {}
     std::string m_name;
     std::vector<std::string> m_argtypes;
+    std::vector<std::string> m_argdefaults;
     std::string m_returntype;
     EMethodType m_type;
 };
@@ -376,6 +377,13 @@
         PUBLIC_CPPYY_STATIC_DATA(enum,    CppyyTestData::EWhat);
         PUBLIC_CPPYY_STATIC_DATA(voidp,   void*);
 
+      // default tester for long double
+        argtypes.clear();
+        argtypes.push_back("long double");
+        methods.push_back(new Cppyy_PseudoMethodInfo("get_ldouble_def", argtypes, "long double"));
+        methods.back()->m_argdefaults.push_back("aap_t(1)");
+        s_methods["CppyyTestData::get_ldouble_def"] = methods.back();
+
       // pretend enum values
         data.push_back(Cppyy_PseudoDatambrInfo(
             "kNothing", "CppyyTestData::EWhat", (ptrdiff_t)&Pseudo_kNothing, true));
@@ -416,6 +424,8 @@
 char* cppyy_resolve_name(const char* cppitem_name) {
     if (cppyy_is_enum(cppitem_name))
         return cppstring_to_cstring("internal_enum_type_t");
+    else if (strcmp(cppitem_name, "aap_t") == 0)
+        return cppstring_to_cstring("long double");
     return cppstring_to_cstring(cppitem_name);
 }
 
@@ -520,6 +530,12 @@
     } else if (idx == s_methods["CppyyTestData::set_double_cr"]) {
         assert(self && nargs == 1);
         ((dummy::CppyyTestData*)self)->set_double_cr(*(double*)&((CPPYY_G__value*)args)[0]);
+    } else if (idx == s_methods["CppyyTestData::set_ldouble"]) {
+        assert(self && nargs == 1);
+        ((dummy::CppyyTestData*)self)->set_ldouble(((CPPYY_G__value*)args)[0].obj.ld);
+    } else if (idx == s_methods["CppyyTestData::set_ldouble_cr"]) {
+        assert(self && nargs == 1);
+        ((dummy::CppyyTestData*)self)->set_ldouble_cr(*(long double*)&((CPPYY_G__value*)args)[0]);
     } else {
         assert(!"method unknown in cppyy_call_v");
     }
@@ -794,6 +810,26 @@
     return result;
 }
 
+double cppyy_call_nld(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
+    double result = 0.;
+    Cppyy_PseudoMethodInfo* idx = (Cppyy_PseudoMethodInfo*)method;
+    if (idx == s_methods["CppyyTestData::get_ldouble_def"]) {
+        if (nargs == 1)
+            result = (double)((dummy::CppyyTestData*)self)->get_ldouble_def(
+                ((CPPYY_G__value*)args)[0].obj.ld);
+        else {
+            assert(self && nargs == 0);
+            result = (double)((dummy::CppyyTestData*)self)->get_ldouble_def();
+        }
+    } else if (idx == s_methods["CppyyTestData::get_ldouble"]) {
+        assert(self && nargs == 0);
+        result = (double)((dummy::CppyyTestData*)self)->get_ldouble();
+    } else {
+        assert(!"method unknown in cppyy_call_nld");
+    }
+    return result;
+}
+
 #define DISPATCH_CALL_R_GET(tpname)                                           \
     else if (idx == s_methods["CppyyTestData::get_"#tpname"_r"]) {            \
         assert(self && nargs == 0);                                           \
@@ -966,14 +1002,16 @@
 }
 
 int cppyy_method_req_args(cppyy_method_t method) {
-    return cppyy_method_num_args(method);
+    return cppyy_method_num_args(method)-((Cppyy_PseudoMethodInfo*)method)->m_argdefaults.size();
 }
 
 char* cppyy_method_arg_type(cppyy_method_t method, int idx) {
     return cppstring_to_cstring(((Cppyy_PseudoMethodInfo*)method)->m_argtypes[idx]);
 }
 
-char* cppyy_method_arg_default(cppyy_method_t, int /* arg_index */) {
+char* cppyy_method_arg_default(cppyy_method_t method, int idx) {
+    if (idx < (int)((Cppyy_PseudoMethodInfo*)method)->m_argdefaults.size())
+        return cppstring_to_cstring(((Cppyy_PseudoMethodInfo*)method)->m_argdefaults[idx]);
     return cppstring_to_cstring("");
 }
 
@@ -1094,13 +1132,30 @@
 }
 
 cppyy_object_t cppyy_charp2stdstring(const char* str, size_t sz) {
-    void* arena = new char[sz];
-    new (arena) std::string(str, sz);
-    return (cppyy_object_t)arena;
+    return (cppyy_object_t)new std::string(str, sz);
+}
+
+const char* cppyy_stdstring2charp(cppyy_object_t ptr, size_t* lsz) {
+    *lsz = ((std::string*)ptr)->size();
+    return ((std::string*)ptr)->data();
 }
 
 cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr) {
-    void* arena = new char[sizeof(std::string)];
-    new (arena) std::string(*(std::string*)ptr);
-    return (cppyy_object_t)arena;
+    return (cppyy_object_t)new std::string(*(std::string*)ptr);
 }
+
+double cppyy_longdouble2double(void* p) {
+    return (double)*(long double*)p;
+}
+
+void cppyy_double2longdouble(double d, void* p) {
+    *(long double*)p = d;
+}
+
+int cppyy_vectorbool_getitem(cppyy_object_t ptr, int idx) {
+    return (int)(*(std::vector<bool>*)ptr)[idx];
+}
+
+void cppyy_vectorbool_setitem(cppyy_object_t ptr, int idx, int value) {
+    (*(std::vector<bool>*)ptr)[idx] = (bool)value;
+}
diff --git a/pypy/module/_cppyy/test/datatypes.cxx b/pypy/module/_cppyy/test/datatypes.cxx
--- a/pypy/module/_cppyy/test/datatypes.cxx
+++ b/pypy/module/_cppyy/test/datatypes.cxx
@@ -113,6 +113,7 @@
 float                CppyyTestData::get_float()   { return m_float; }
 double               CppyyTestData::get_double()  { return m_double; }
 long double          CppyyTestData::get_ldouble() { return m_ldouble; }
+long double          CppyyTestData::get_ldouble_def(long double ld) { return ld; }
 CppyyTestData::EWhat CppyyTestData::get_enum()    { return m_enum; }
 void*                CppyyTestData::get_voidp()   { return m_voidp; }
 
diff --git a/pypy/module/_cppyy/test/datatypes.h b/pypy/module/_cppyy/test/datatypes.h
--- a/pypy/module/_cppyy/test/datatypes.h
+++ b/pypy/module/_cppyy/test/datatypes.h
@@ -32,6 +32,8 @@
         enum    {E1 = -1};
         enum EE {E2 = -1};
     };
+
+    typedef enum { AA = 1, BB, CC, DD } letter_code;
 }
 
 
@@ -94,6 +96,8 @@
     float                get_float();
     double               get_double();
     long double          get_ldouble();
+    typedef long double aap_t;
+    long double          get_ldouble_def(long double ld = aap_t(1));
     EWhat                get_enum();
     void*                get_voidp();
 
diff --git a/pypy/module/_cppyy/test/datatypes.xml b/pypy/module/_cppyy/test/datatypes.xml
--- a/pypy/module/_cppyy/test/datatypes.xml
+++ b/pypy/module/_cppyy/test/datatypes.xml
@@ -6,6 +6,7 @@
   <enum name="EFruit" />
   <enum name="EnumSpace::E" />
   <class name="EnumSpace::EnumClass" />
+  <enum name="EnumSpace::letter_code" />
 
   <function pattern="get_*" />
   <function pattern="set_*" />
diff --git a/pypy/module/_cppyy/test/stltypes.h b/pypy/module/_cppyy/test/stltypes.h
--- a/pypy/module/_cppyy/test/stltypes.h
+++ b/pypy/module/_cppyy/test/stltypes.h
@@ -4,12 +4,23 @@
 #include <utility>
 #include <vector>
 
+
 //- basic example class
 class just_a_class {
 public:
     int m_i;
 };
 
+// enum for vector of enums setitem tests
+enum VecTestEnum {
+    EVal1 = 1, EVal2 = 3
+};
+
+namespace VecTestEnumNS {
+    enum VecTestEnum { EVal1 = 5, EVal2 = 42 };
+}
+
+
 //- class with lots of std::string handling
 class stringy_class {
 public:
@@ -31,35 +42,15 @@
 class stl_like_class {
 public: 
    no_dict_available* begin() { return 0; }
-   no_dict_available* end() { return 0; }
+   no_dict_available* end() { return (no_dict_available*)1; }
    int size() { return 4; }
    int operator[](int i) { return i; }
    std::string operator[](double) { return "double"; }
    std::string operator[](const std::string&) { return "string"; }
 };      
 
-
-//- instantiations of used STL types
 namespace {
-
     stl_like_class<int> stlc_1;
-
-} // unnamed namespace
-
-
-// comps for int only to allow testing: normal use of vector is looping over a
-// range-checked version of __getitem__
-#if defined(__clang__) && defined(__APPLE__)
-namespace std {
-#define ns_prefix std::
-#elif defined(__GNUC__) || defined(__GNUG__)
-namespace __gnu_cxx {
-#define ns_prefix
-#endif
-extern template bool ns_prefix operator==(const std::vector<int>::iterator&,
-                         const std::vector<int>::iterator&);
-extern template bool ns_prefix operator!=(const std::vector<int>::iterator&,
-                         const std::vector<int>::iterator&);
 }
 
 
@@ -67,6 +58,8 @@
 namespace ArrayTest {
 
 struct Point {
+    Point() : px(0), py(0) {}
+    Point(int x, int y) : px(x), py(y) {}
     int px, py;
 };
 
diff --git a/pypy/module/_cppyy/test/stltypes.xml b/pypy/module/_cppyy/test/stltypes.xml
--- a/pypy/module/_cppyy/test/stltypes.xml
+++ b/pypy/module/_cppyy/test/stltypes.xml
@@ -1,6 +1,9 @@
 <lcgdict>
 
   <class name="just_a_class" />
+  <enum name="VecTestEnum" />
+  <namespace name="VecTestEnumNS" />
+  <enum name="VecTestEnumNS::VecTestEnum" />
   <class name="stringy_class" />
   <class pattern="stl_like_class<*>" />
 
diff --git a/pypy/module/_cppyy/test/test_datatypes.py b/pypy/module/_cppyy/test/test_datatypes.py
--- a/pypy/module/_cppyy/test/test_datatypes.py
+++ b/pypy/module/_cppyy/test/test_datatypes.py
@@ -56,10 +56,11 @@
         assert round(c.m_double         + 77., 11) == 0
         assert round(c.get_double_cr()  + 77., 11) == 0
         assert round(c.get_double_r()   + 77., 11) == 0
-        #assert round(c.m_ldouble        + 88., 24) == 0
-        #assert round(c.get_ldouble_cr() + 88., 24) == 0
-        #assert round(c.get_ldouble_r()  + 88., 24) == 0
-        assert round(c.m_double + 77., 8) == 0
+        assert round(c.m_ldouble        + 88., 24) == 0
+        assert round(c.get_ldouble_cr() + 88., 24) == 0
+        assert round(c.get_ldouble_r()  + 88., 24) == 0
+        assert round(c.get_ldouble_def()  -1., 24) == 0
+        assert round(c.get_ldouble_def(2) -2., 24) == 0
 
         """# complex<double> type
         assert type(c.get_complex()) == complex
@@ -187,16 +188,20 @@
             assert eval('c.m_%s' % names[i]) == 3*i
 
         # float types through functions
-        c.set_float( 0.123 );  assert round(c.get_float()  - 0.123, 5) == 0
-        c.set_double( 0.456 ); assert round(c.get_double() - 0.456, 8) == 0
+        c.set_float(0.123);   assert round(c.get_float()   - 0.123, 5) == 0
+        c.set_double(0.456);  assert round(c.get_double()  - 0.456, 8) == 0
+        c.set_ldouble(0.789); assert round(c.get_ldouble() - 0.789, 8) == 0
 
         # float types through data members
-        c.m_float = 0.123;      assert round(c.get_float()  - 0.123, 5) == 0
-        c.set_float(0.234);     assert round(c.m_float      - 0.234, 5) == 0
-        c.set_float_cr(0.456);  assert round(c.m_float      - 0.456, 5) == 0
-        c.m_double = 0.678;     assert round(c.get_double() - 0.678, 8) == 0
-        c.set_double(0.890);    assert round(c.m_double     - 0.890, 8) == 0
-        c.set_double_cr(0.012); assert round(c.m_double     - 0.012, 8) == 0
+        c.m_float = 0.123;       assert round(c.get_float()   - 0.123, 5) == 0
+        c.set_float(0.234);      assert round(c.m_float       - 0.234, 5) == 0
+        c.set_float_cr(0.456);   assert round(c.m_float       - 0.456, 5) == 0
+        c.m_double = 0.678;      assert round(c.get_double()  - 0.678, 8) == 0
+        c.set_double(0.890);     assert round(c.m_double      - 0.890, 8) == 0
+        c.set_double_cr(0.012);  assert round(c.m_double      - 0.012, 8) == 0
+        c.m_ldouble = 0.876;     assert round(c.get_ldouble() - 0.876, 8) == 0
+        c.set_ldouble(0.098);    assert round(c.m_ldouble     - 0.098, 8) == 0
+        c.set_ldouble_cr(0.210); assert round(c.m_ldouble     - 0.210, 8) == 0
 
         # arrays; there will be pointer copies, so destroy the current ones
         c.destroy_arrays()
@@ -295,10 +300,12 @@
         assert CppyyTestData.s_ullong   ==  404
 
         # floating point types
-        assert round(CppyyTestData.s_float  + 606., 5) == 0
-        assert round(c.s_float              + 606., 5) == 0
-        assert round(CppyyTestData.s_double + 707., 8) == 0
-        assert round(c.s_double             + 707., 8) == 0
+        assert round(CppyyTestData.s_float   + 606., 5) == 0
+        assert round(c.s_float               + 606., 5) == 0
+        assert round(CppyyTestData.s_double  + 707., 8) == 0
+        assert round(c.s_double              + 707., 8) == 0
+        assert round(CppyyTestData.s_ldouble + 808., 8) == 0
+        assert round(c.s_ldouble             + 808., 8) == 0
 
         c.__destruct__()
 
@@ -363,6 +370,10 @@
         assert CppyyTestData.s_double             == -math.pi
         CppyyTestData.s_double                     =  math.pi
         assert c.s_double                         ==  math.pi
+        c.s_ldouble                                = -math.pi
+        assert CppyyTestData.s_ldouble            == -math.pi
+        CppyyTestData.s_ldouble                    =  math.pi
+        assert c.s_ldouble                        ==  math.pi
 
         c.__destruct__()
 
@@ -499,6 +510,11 @@
         assert gbl.EnumSpace.EnumClass.E1 == -1   # anonymous
         assert gbl.EnumSpace.EnumClass.E2 == -1   # named type
 
+        # typedef enum
+        assert gbl.EnumSpace.letter_code
+        assert gbl.EnumSpace.AA == 1
+        assert gbl.EnumSpace.BB == 2
+
     def test11_string_passing(self):
         """Test passing/returning of a const char*"""
 
diff --git a/pypy/module/_cppyy/test/test_stltypes.py b/pypy/module/_cppyy/test/test_stltypes.py
--- a/pypy/module/_cppyy/test/test_stltypes.py
+++ b/pypy/module/_cppyy/test/test_stltypes.py
@@ -215,6 +215,26 @@
         assert len(vb[4:8]) == 4
         assert list(vb[4:8]) == [False]*3+[True]
 
+    def test08_vector_enum(self):
+        """Usability of std::vector<> of some enums"""
+
+        import _cppyy as cppyy
+
+        # TODO: would like to use cppyy.gbl.VecTestEnum but that's an int
+        assert cppyy.gbl.VecTestEnum
+        ve = cppyy.gbl.std.vector['VecTestEnum']()
+        ve.push_back(cppyy.gbl.EVal1);
+        assert ve[0] == 1
+        ve[0] = cppyy.gbl.EVal2
+        assert ve[0] == 3
+
+        assert cppyy.gbl.VecTestEnumNS.VecTestEnum
+        ve = cppyy.gbl.std.vector['VecTestEnumNS::VecTestEnum']()
+        ve.push_back(cppyy.gbl.VecTestEnumNS.EVal1);
+        assert ve[0] == 5
+        ve[0] = cppyy.gbl.VecTestEnumNS.EVal2
+        assert ve[0] == 42
+
 
 class AppTestSTLSTRING:
     spaceconfig = dict(usemodules=['_cppyy', '_rawffi', 'itertools'])
@@ -484,10 +504,11 @@
         stl_like_class = cppyy.gbl.stl_like_class
 
         a = stl_like_class(int)()
-        for i in a:
-            pass
+        assert len(a) == 4
+        for i, j in enumerate(a):
+            assert i == j
 
-        assert i == 3
+        assert i == len(a)-1
 
 
 class AppTestSTLITERATOR:
@@ -594,9 +615,9 @@
     def test03_array_of_pointer_to_pods(self):
         """Usage of std::array of pointer to PODs"""
 
-        import cppyy
-        from cppyy import gbl
-        from cppyy.gbl import std
+        import _cppyy as cppyy
+        gbl = cppyy.gbl
+        std = cppyy.gbl.std
 
         ll = [gbl.ArrayTest.Point() for i in range(4)]
         for i in range(len(ll)):

From pypy.commits at gmail.com  Sat Aug  4 18:01:28 2018
From: pypy.commits at gmail.com (mattip)
Date: Sat, 04 Aug 2018 15:01:28 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: return utf8, len,
 pos from decoders
Message-ID: <5b662238.1c69fb81.807a.ccad@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94937:8364a4fb4acd
Date: 2018-07-31 14:17 -0700
http://bitbucket.org/pypy/pypy/changeset/8364a4fb4acd/

Log:	return utf8, len, pos from decoders

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -152,10 +152,9 @@
     return result_utf8, length
 
 def decode_raw_unicode_escape(space, string):
-    result_utf8, lgt = str_decode_raw_unicode_escape(
+    return str_decode_raw_unicode_escape(
         string, "strict",
         final=True, errorhandler=decode_error_handler(space))
-    return result_utf8, lgt
 
 def check_ascii_or_raise(space, string):
     try:
@@ -181,7 +180,7 @@
 def str_decode_ascii(s, errors, final, errorhandler):
     try:
         rutf8.check_ascii(s)
-        return s, len(s)
+        return s, len(s), len(s)
     except rutf8.CheckError:
         return _str_decode_ascii_slowpath(s, errors, final, errorhandler)
 
@@ -199,12 +198,12 @@
             i += 1
     ress = res.build()
     lgt = rutf8.check_utf8(ress, True)
-    return ress, lgt
+    return ress, lgt, lgt
 
 def str_decode_latin_1(s, errors, final, errorhandler):
     try:
         rutf8.check_ascii(s)
-        return s, len(s)
+        return s, len(s), len(s)
     except rutf8.CheckError:
         return _str_decode_latin_1_slowpath(s, errors, final, errorhandler)
 
@@ -224,7 +223,7 @@
             res.append_slice(s, start, end)
             i = end
     # cannot be ASCII, cannot have surrogates, I believe
-    return res.build(), len(s)
+    return res.build(), len(s), len(s)
 
 def utf8_encode_latin_1(s, errors, errorhandler):
     try:
@@ -430,7 +429,7 @@
         res.append(r)
 
     r = res.build()
-    return r, rutf8.check_utf8(r, True)
+    return r, rutf8.check_utf8(r, True), pos
 
 hexdigits = "0123456789ABCDEFabcdef"
 
@@ -650,7 +649,7 @@
         pos = hexescape(builder, s, pos, digits,
                            "rawunicodeescape", errorhandler, message, errors)
 
-    return builder.build(), builder.getlength()
+    return builder.build(), builder.getlength(), pos
 
 _utf8_encode_unicode_escape = rutf8.make_utf8_escape_function()
 
@@ -785,7 +784,7 @@
                      errorhandler=None):
     size = len(s)
     if size == 0:
-        return '', 0
+        return '', 0, 0
 
     inShift = False
     base64bits = 0
@@ -920,7 +919,7 @@
         final_length = shiftOutStartPos # back off output
 
     assert final_length >= 0
-    return result.build()[:final_length], outsize
+    return result.build()[:final_length], outsize, size
 
 def utf8_encode_utf_7(s, errors, errorhandler):
     size = len(s)
@@ -1010,21 +1009,15 @@
 
 def str_decode_utf_16(s, errors, final=True,
                       errorhandler=None):
-    result, lgt = str_decode_utf_16_helper(s, errors, final,
-                                                         errorhandler, "native")
-    return result, lgt
+    return str_decode_utf_16_helper(s, errors, final, errorhandler, "native")
 
 def str_decode_utf_16_be(s, errors, final=True,
                         errorhandler=None):
-    result, lgt = str_decode_utf_16_helper(s, errors, final,
-                                                         errorhandler, "big")
-    return result, lgt
+    return str_decode_utf_16_helper(s, errors, final, errorhandler, "big")
 
 def str_decode_utf_16_le(s, errors, final=True,
                          errorhandler=None):
-    result, lgt = str_decode_utf_16_helper(s, errors, final,
-                                                         errorhandler, "little")
-    return result, lgt
+    return str_decode_utf_16_helper(s, errors, final, errorhandler, "little")
 
 def str_decode_utf_16_helper(s, errors, final=True,
                              errorhandler=None,
@@ -1067,7 +1060,7 @@
     else:
         bo = 1
     if size == 0:
-        return '', 0
+        return '', 0, 0
     if bo == -1:
         # force little endian
         ihi = 1
@@ -1127,7 +1120,7 @@
             result.append(r)
     r = result.build()
     lgt = rutf8.check_utf8(r, True)
-    return result.build(), lgt
+    return result.build(), lgt, pos
 
 def _STORECHAR(result, CH, byteorder):
     hi = chr(((CH) >> 8) & 0xff)
@@ -1235,24 +1228,21 @@
 
 def str_decode_utf_32(s, errors, final=True,
                       errorhandler=None):
-    result, lgt = str_decode_utf_32_helper(
+    return str_decode_utf_32_helper(
         s, errors, final, errorhandler, "native", 'utf-32-' + BYTEORDER2,
         allow_surrogates=False)
-    return result, lgt
 
 def str_decode_utf_32_be(s, errors, final=True,
                          errorhandler=None):
-    result, lgt = str_decode_utf_32_helper(
+    return str_decode_utf_32_helper(
         s, errors, final, errorhandler, "big", 'utf-32-be',
         allow_surrogates=False)
-    return result, lgt
 
 def str_decode_utf_32_le(s, errors, final=True,
                          errorhandler=None):
-    result, lgt = str_decode_utf_32_helper(
+    return str_decode_utf_32_helper(
         s, errors, final, errorhandler, "little", 'utf-32-le',
         allow_surrogates=False)
-    return result, lgt
 
 BOM32_DIRECT  = intmask(0x0000FEFF)
 BOM32_REVERSE = intmask(0xFFFE0000)
@@ -1300,7 +1290,7 @@
     else:
         bo = 1
     if size == 0:
-        return '', 0
+        return '', 0, 0
     if bo == -1:
         # force little endian
         iorder = [0, 1, 2, 3]
@@ -1342,7 +1332,7 @@
         pos += 4
     r = result.build()
     lgt = rutf8.check_utf8(r, True)
-    return r, lgt
+    return r, lgt, pos
 
 def _STORECHAR32(result, CH, byteorder):
     c0 = chr(((CH) >> 24) & 0xff)
@@ -1615,7 +1605,7 @@
         pos += 1
     r = result.build()
     lgt = rutf8.codepoints_in_utf8(r)
-    return r, lgt
+    return r, lgt, pos
 
 def utf8_encode_charmap(s, errors, errorhandler=None, mapping=None):
     if mapping is None:
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -50,7 +50,7 @@
             else:
                 w_cls = space.w_UnicodeEncodeError
                 length = rutf8.codepoints_in_utf8(input)
-                w_input = space.newtext((input, length))
+                w_input = space.newtext((input, length, length))
             w_exc =  space.call_function(
                 w_cls,
                 space.newtext(encoding),
@@ -266,7 +266,6 @@
 
 def backslashreplace_errors(space, w_exc):
 
-    import pdb;pdb.set_trace()
     check_exception(space, w_exc)
     if (space.isinstance_w(w_exc, space.w_UnicodeEncodeError) or
         space.isinstance_w(w_exc, space.w_UnicodeTranslateError)):
@@ -650,8 +649,9 @@
             errors = 'strict'
         final = space.is_true(w_final)
         state = space.fromcache(CodecState)
-        result, length = func(string, errors, final, state.decode_error_handler)
-        return space.newtuple([space.newutf8(result, length), space.newint(length)])
+        result, length, pos = func(string, errors, final, state.decode_error_handler)
+        # must return bytes, len_of_original_string
+        return space.newtuple([space.newutf8(result, length), space.newint(pos)])
     wrap_decoder.__name__ = func.__name__
     globals()[name] = wrap_decoder
 
@@ -705,11 +705,11 @@
             errors = 'strict'
         final = space.is_true(w_final)
         state = space.fromcache(CodecState)
-        result, length = runicode.str_decode_mbcs(
+        result, length, pos = runicode.str_decode_mbcs(
             string, len(string), errors,
             final, state.decode_error_handler,
             force_ignore=False)
-        return space.newtuple([space.newtext(result, length), space.newint(length)])
+        return space.newtuple([space.newtext(result, length), space.newint(pos)])
 
 # utf-8 functions are not regular, because we have to pass
 # "allow_surrogates=False"
@@ -739,7 +739,7 @@
     try:
         lgt = rutf8.check_utf8(string, allow_surrogates=True)
     except rutf8.CheckError:
-        res, lgt = unicodehelper.str_decode_utf8(string,
+        res, lgt, pos = unicodehelper.str_decode_utf8(string,
             errors, final, state.decode_error_handler)
         return space.newtuple([space.newutf8(res, lgt),
                                space.newint(lgt)])
@@ -762,7 +762,7 @@
         byteorder = 'little'
     else:
         byteorder = 'big'
-    res, lgt = str_decode_utf_16_helper(
+    res, lgt, pos = str_decode_utf_16_helper(
         data, errors, final,
         state.decode_error_handler, byteorder)
     return space.newtuple([space.newutf8(res, lgt),
@@ -781,7 +781,7 @@
         byteorder = 'little'
     else:
         byteorder = 'big'
-    res, lgt = str_decode_utf_32_helper(
+    res, lgt, pos = str_decode_utf_32_helper(
         data, errors, final,
         state.decode_error_handler, byteorder)
     return space.newtuple([space.newutf8(res, lgt),
@@ -882,7 +882,7 @@
 
     final = True
     state = space.fromcache(CodecState)
-    result, lgt = unicodehelper.str_decode_charmap(
+    result, lgt, pos = unicodehelper.str_decode_charmap(
         string, errors, final, state.decode_error_handler, mapping)
     return space.newtuple([space.newutf8(result, lgt),
                            space.newint(len(string))])
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -142,14 +142,10 @@
         assert decode(br"[\x0]\x0", "replace") == (b"[?]?", 8)
 
     def test_unicode_escape(self):
-        import sys
         from _codecs import unicode_escape_encode, unicode_escape_decode
         assert unicode_escape_encode('abc') == ('abc'.encode('unicode_escape'), 3)
         assert unicode_escape_decode(b'abc') == (b'abc'.decode('unicode_escape'), 3)
-        if sys.version_info[0] < 3:
-            lgt = 12
-        else:
-            lgt = 3
+        lgt = 12
         assert unicode_escape_decode(b'\\x61\\x62\\x63') == ('abc', lgt)
 
 
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -384,10 +384,10 @@
         if isinstance(s, unicode):
             s, lgt = s.encode('utf8'), len(s)
         elif isinstance(s, str):
-            s, lgt = decode_utf8sp(self, s)
+            s, lgt, codepoints = decode_utf8sp(self, s)
         elif isinstance(s, tuple):
             # result of decode_utf8
-            s, lgt = s
+            s, lgt, codepoints = s
         else:
             # XXX what is s ?
             lgt = rutf8.check_utf8(s, True)
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -1185,14 +1185,6 @@
 
 def encode_object(space, w_object, encoding, errors):
     utf8 = space.utf8_w(w_object)
-    idx = rutf8.surrogate_in_utf8(utf8)
-    if idx >= 0:
-        print 'surrogate in unicodeobject.encode_object(', w_object, ',', encoding, ',', errors, ')', 'raising'
-        if errors is None:
-            w_err_handler = unicodehelper.encode_error_handler(space)
-        else:
-            w_err_handler = lookup_error(space, errors)
-        w_err_handler(None, "utf8", "surrogates not allowed", utf8, idx, idx + 1)
     if errors is None or errors == 'strict':
         if encoding is None or encoding == 'utf-8':
             #if rutf8.has_surrogates(utf8):

From pypy.commits at gmail.com  Sat Aug  4 18:01:31 2018
From: pypy.commits at gmail.com (mattip)
Date: Sat, 04 Aug 2018 15:01:31 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: fixes for translation,
 surrogates
Message-ID: <5b66223b.1c69fb81.2fa4c.c421@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94938:d7b217949b58
Date: 2018-08-03 20:18 -0700
http://bitbucket.org/pypy/pypy/changeset/d7b217949b58/

Log:	fixes for translation, surrogates

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1721,9 +1721,9 @@
         return w_obj.convert_to_w_unicode(self)
 
     def realunicode_w(self, w_obj):
-        from rpython.rlib.runicode import str_decode_utf_8
+        from pypy.interpreter.unicodehelper import decode_utf8sp
         utf8 = self.utf8_w(w_obj)
-        return str_decode_utf_8(utf8, len(utf8), 'strict', True)[0]
+        return decode_utf8sp(self, utf8)[0]
 
     def utf8_0_w(self, w_obj):
         "Like utf8_w, but rejects strings with NUL bytes."
@@ -1763,7 +1763,8 @@
     def realutf8_w(self, w_obj):
         # Like utf8_w(), but only works if w_obj is really of type
         # 'unicode'.  On Python 3 this is the same as utf8_w().
-        if not self.isinstance_w(w_obj, self.w_unicode):
+        from pypy.objspace.std.unicodeobject import W_UnicodeObject
+        if not isinstance(w_obj, W_UnicodeObject):
             raise oefmt(self.w_TypeError, "argument must be a unicode")
         return self.utf8_w(w_obj)
 
@@ -1784,7 +1785,7 @@
     def fsdecode_w(self, w_obj):
         if self.isinstance_w(w_obj, self.w_bytes):
             w_obj = self.fsdecode(w_obj)
-        return self.unicode0_w(w_obj)
+        return self.utf8_w(w_obj)
 
     def bool_w(self, w_obj):
         # Unwraps a bool, also accepting an int for compatibility.
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -10,7 +10,6 @@
 from rpython.rlib.objectmodel import dont_inline, not_rpython
 from rpython.rlib import rstack, rstackovf
 from rpython.rlib import rwin32
-from rpython.rlib import runicode
 
 from pypy.interpreter import debug
 
@@ -518,7 +517,10 @@
                     elif fmt == 'N':
                         result = value.getname(space)
                     elif fmt == '8':
-                        result = _decode_utf8(value)
+                        try:
+                            result = value.decode('utf8')
+                        except UnicodeDecodeError:
+                            result = value.decode('unicode-escape')
                     else:
                         if isinstance(value, unicode):
                             result = value
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -45,7 +45,8 @@
                  closure=None, w_ann=None, forcename=None, qualname=None):
         self.space = space
         self.name = forcename or code.co_name
-        self.qualname = qualname or self.name.decode('utf-8')
+        self.qualname = qualname or self.name
+        assert isinstance(self.qualname, str)
         self.w_doc = None   # lazily read from code.getdocstring()
         self.code = code       # Code instance
         self.w_func_globals = w_globals  # the globals dictionary
@@ -434,7 +435,7 @@
 
     def fset_func_qualname(self, space, w_name):
         try:
-            self.qualname = space.utf8_w(w_name)
+            self.qualname = space.realutf8_w(w_name)
         except OperationError as e:
             if e.match(space, space.w_TypeError):
                 raise oefmt(space.w_TypeError,
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -30,6 +30,7 @@
     return raise_unicode_exception_decode
 
 def decode_never_raise(errors, encoding, msg, s, startingpos, endingpos):
+    assert startingpos >= 0
     ux = ['\ux' + hex(ord(x))[2:].upper() for x in s[startingpos:endingpos]]
     return ''.join(ux), endingpos, endingpos
 
@@ -116,7 +117,7 @@
         # instead
         from pypy.module._codecs.locale import (
             unicode_encode_locale_surrogateescape)
-        uni = space.realunicode_w(w_uni)
+        uni = space.realunicode_w(w_uni).decode('utf8')
         if u'\x00' in uni:
             raise oefmt(space.w_ValueError, "embedded null character")
         bytes = unicode_encode_locale_surrogateescape(
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -1164,7 +1164,8 @@
     unerase = staticmethod(unerase)
 
     def wrap(self, unwrapped):
-        return self.space.newutf8(unwrapped, len(unwrapped))
+        return self.space.newutf8(unwrapped,
+                                  rutf8.codepoints_in_utf8(unwrapped))
 
     def unwrap(self, wrapped):
         return self.space.utf8_w(wrapped)
@@ -1209,7 +1210,7 @@
     ##     return self.space.newlist_bytes(self.listview_bytes(w_dict))
 
     def wrapkey(space, key):
-        return space.newutf8(key, len(key))
+        return space.newutf8(key, rutf8.codepoints_in_utf8(key))
 
     @jit.look_inside_iff(lambda self, w_dict:
                          w_dict_unrolling_heuristic(w_dict))

From pypy.commits at gmail.com  Sat Aug  4 18:01:33 2018
From: pypy.commits at gmail.com (mattip)
Date: Sat, 04 Aug 2018 15:01:33 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: make realunicode_w return
 rpython unicode
Message-ID: <5b66223d.1c69fb81.99185.55d0@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94939:ba04669afe92
Date: 2018-08-04 14:52 -0700
http://bitbucket.org/pypy/pypy/changeset/ba04669afe92/

Log:	make realunicode_w return rpython unicode

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1723,7 +1723,7 @@
     def realunicode_w(self, w_obj):
         from pypy.interpreter.unicodehelper import decode_utf8sp
         utf8 = self.utf8_w(w_obj)
-        return decode_utf8sp(self, utf8)[0]
+        return decode_utf8sp(self, utf8)[0].decode('utf8')
 
     def utf8_0_w(self, w_obj):
         "Like utf8_w, but rejects strings with NUL bytes."

From pypy.commits at gmail.com  Sat Aug  4 18:01:34 2018
From: pypy.commits at gmail.com (mattip)
Date: Sat, 04 Aug 2018 15:01:34 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: decode returns a triplet
 (needed by implement_codecs.py)
Message-ID: <5b66223e.1c69fb81.b4ac1.c5ee@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94940:418b72a670e7
Date: 2018-08-04 14:53 -0700
http://bitbucket.org/pypy/pypy/changeset/418b72a670e7/

Log:	decode returns a triplet (needed by implement_codecs.py)

diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1,5 +1,6 @@
 from __future__ import division
 import py, sys
+from pytest import raises
 from pypy.interpreter.astcompiler import codegen, astbuilder, symtable, optimize
 from pypy.interpreter.pyparser import pyparse
 from pypy.interpreter.pyparser.test import expressions
diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -95,7 +95,7 @@
             unicodehelper.check_utf8_or_raise(space, s, ps, q)
             substr = decode_unicode_utf8(space, s, ps, q)
         r = unicodehelper.decode_unicode_escape(space, substr)
-        v, length = r
+        v, length, pos = r
         return space.newutf8(v, length)
 
     assert 0 <= ps <= q
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -145,12 +145,11 @@
     from pypy.module._codecs import interp_codecs
     state = space.fromcache(interp_codecs.CodecState)
     unicodedata_handler = state.get_unicodedata_handler(space)
-    result_utf8, length = str_decode_unicode_escape(
+    return str_decode_unicode_escape(
         string, "strict",
         final=True,
         errorhandler=state.decode_error_handler,
         ud_handler=unicodedata_handler)
-    return result_utf8, length
 
 def decode_raw_unicode_escape(space, string):
     return str_decode_raw_unicode_escape(
@@ -469,7 +468,7 @@
 def str_decode_unicode_escape(s, errors, final, errorhandler, ud_handler):
     size = len(s)
     if size == 0:
-        return '', 0
+        return '', 0, 0
 
     builder = rutf8.Utf8StringBuilder(size)
     pos = 0
@@ -588,7 +587,7 @@
             builder.append_char('\\')
             builder.append_code(ord(ch))
 
-    return builder.build(), builder.getlength()
+    return builder.build(), builder.getlength(), pos
 
 def wcharpsize2utf8(space, wcharp, size):
     """Safe version of rffi.wcharpsize2utf8.

From pypy.commits at gmail.com  Sat Aug  4 18:01:36 2018
From: pypy.commits at gmail.com (mattip)
Date: Sat, 04 Aug 2018 15:01:36 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: fix d7b217949b58
Message-ID: <5b662240.1c69fb81.77b97.59eb@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94941:fe6bd08c8ad1
Date: 2018-08-04 14:55 -0700
http://bitbucket.org/pypy/pypy/changeset/fe6bd08c8ad1/

Log:	fix d7b217949b58

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -517,10 +517,10 @@
                     elif fmt == 'N':
                         result = value.getname(space)
                     elif fmt == '8':
-                        try:
-                            result = value.decode('utf8')
-                        except UnicodeDecodeError:
-                            result = value.decode('unicode-escape')
+                        if isinstance(value, unicode):
+                            result = value.encode('utf8')
+                        else:
+                            result = value
                     else:
                         if isinstance(value, unicode):
                             result = value

From pypy.commits at gmail.com  Sat Aug  4 18:01:38 2018
From: pypy.commits at gmail.com (mattip)
Date: Sat, 04 Aug 2018 15:01:38 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: disallow encoding with
 surrogates, occured in pyparsing
Message-ID: <5b662242.1c69fb81.2fd3a.be82@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94942:d5aecbf7948c
Date: 2018-08-04 14:59 -0700
http://bitbucket.org/pypy/pypy/changeset/d5aecbf7948c/

Log:	disallow encoding with surrogates, occured in pyparsing

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -1185,6 +1185,14 @@
 
 def encode_object(space, w_object, encoding, errors):
     utf8 = space.utf8_w(w_object)
+    # TODO: refactor unnatrual use of error hanlders here,
+    # we should make a single pass over the utf8 str
+    pos = rutf8.surrogate_in_utf8(utf8)
+    if pos >= 0:
+        eh = unicodehelper.encode_error_handler(space)
+        eh(None, "utf8", "surrogates not allowed", utf8,
+            pos, pos + 1)
+        assert False, "always raises"
     if errors is None or errors == 'strict':
         if encoding is None or encoding == 'utf-8':
             #if rutf8.has_surrogates(utf8):

From pypy.commits at gmail.com  Sat Aug  4 19:45:43 2018
From: pypy.commits at gmail.com (mattip)
Date: Sat, 04 Aug 2018 16:45:43 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: revert rpython changes,
 we should not use runicode
Message-ID: <5b663aa7.1c69fb81.d68c.2177@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94943:01c64e911294
Date: 2018-08-04 16:42 -0700
http://bitbucket.org/pypy/pypy/changeset/01c64e911294/

Log:	revert rpython changes, we should not use runicode

diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -707,7 +707,7 @@
             address.unlock()
             errno = _c.geterrno()
             timeout = self.timeout
-            if (timeout > 0.0 and res < 0 and 
+            if (timeout > 0.0 and res < 0 and
                 errno in (_c.EWOULDBLOCK, _c.WSAEWOULDBLOCK)):
                 tv = rffi.make(_c.timeval)
                 rffi.setintfield(tv, 'c_tv_sec', int(timeout))
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -1401,7 +1401,7 @@
             endinpos += 1
         res, pos = errorhandler(errors, encoding,
                                 message, s, pos-2, endinpos)
-        builder.append(res.decode('utf8'))
+        builder.append(res)
     else:
         try:
             chr = r_uint(int(s[pos:pos+digits], 16))
@@ -1411,7 +1411,7 @@
                 endinpos += 1
             res, pos = errorhandler(errors, encoding,
                                     message, s, pos-2, endinpos)
-            builder.append(res.decode('utf8'))
+            builder.append(res)
         else:
             # when we get here, chr is a 32-bit unicode character
             if chr <= MAXUNICODE:
@@ -1427,7 +1427,7 @@
                 message = "illegal Unicode character"
                 res, pos = errorhandler(errors, encoding,
                                         message, s, pos-2, pos+digits)
-                builder.append(res.decode('utf8'))
+                builder.append(res)
     return pos
 
 def str_decode_unicode_escape(s, size, errors, final=False,
@@ -1708,12 +1708,8 @@
             pos += 1
             continue
 
-        if s[pos] == 'u':
-            digits = 4
-            message = "truncated \\uXXXX escape"
-        else:
-            digits = 8
-            message = "truncated \\UXXXXXXXX escape"
+        digits = 4 if s[pos] == 'u' else 8
+        message = "truncated \\uXXXX"
         pos += 1
         pos = hexescape(result, s, pos, digits,
                         "rawunicodeescape", errorhandler, message, errors)

From pypy.commits at gmail.com  Sat Aug  4 19:45:46 2018
From: pypy.commits at gmail.com (mattip)
Date: Sat, 04 Aug 2018 16:45:46 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: fix some utf8 - unicode
 confusion, edge cases
Message-ID: <5b663aaa.1c69fb81.7d71f.2a3c@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94944:d9685a6896aa
Date: 2018-08-04 16:44 -0700
http://bitbucket.org/pypy/pypy/changeset/d9685a6896aa/

Log:	fix some utf8 - unicode confusion, edge cases

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -602,11 +602,11 @@
 
     def getmsg(self):
         if self.num_kwds == 1:
-            if isinstance(self.kwd_name, str):
-                uname = self.kwd_name.decode('utf8')
+            if isinstance(self.kwd_name, unicode):
+                uname = self.kwd_name.encode('utf8')
             else:
                 uname = self.kwd_name
-            msg = u"got an unexpected keyword argument '%s'" % uname
+            msg = "got an unexpected keyword argument '%s'" % uname
         else:
             msg = "got %d unexpected keyword arguments" % (
                 self.num_kwds)
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -520,7 +520,7 @@
                         if isinstance(value, unicode):
                             result = value.encode('utf8')
                         else:
-                            result = value
+                            result = value.decode('utf8', errors='replace')
                     else:
                         if isinstance(value, unicode):
                             result = value
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -117,7 +117,7 @@
         # instead
         from pypy.module._codecs.locale import (
             unicode_encode_locale_surrogateescape)
-        uni = space.realunicode_w(w_uni).decode('utf8')
+        uni = space.realunicode_w(w_uni)
         if u'\x00' in uni:
             raise oefmt(space.w_ValueError, "embedded null character")
         bytes = unicode_encode_locale_surrogateescape(
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -513,7 +513,7 @@
     def descr_encode(self, space, w_encoding=None, w_errors=None):
         encoding, errors = _get_encoding_and_errors(space, w_encoding,
                                                     w_errors)
-        return encode_object(space, self, encoding, errors)
+        return encode_object(space, self, encoding, errors, allow_surrogates=True)
 
     @unwrap_spec(tabsize=int)
     def descr_expandtabs(self, space, tabsize=8):
@@ -1183,16 +1183,17 @@
     return encoding, errors
 
 
-def encode_object(space, w_object, encoding, errors):
+def encode_object(space, w_object, encoding, errors, allow_surrogates=False):
     utf8 = space.utf8_w(w_object)
     # TODO: refactor unnatrual use of error hanlders here,
     # we should make a single pass over the utf8 str
-    pos = rutf8.surrogate_in_utf8(utf8)
-    if pos >= 0:
-        eh = unicodehelper.encode_error_handler(space)
-        eh(None, "utf8", "surrogates not allowed", utf8,
-            pos, pos + 1)
-        assert False, "always raises"
+    if not allow_surrogates:
+        pos = rutf8.surrogate_in_utf8(utf8)
+        if pos >= 0:
+            eh = unicodehelper.encode_error_handler(space)
+            eh(None, "utf8", "surrogates not allowed", utf8,
+                pos, pos + 1)
+            assert False, "always raises"
     if errors is None or errors == 'strict':
         if encoding is None or encoding == 'utf-8':
             #if rutf8.has_surrogates(utf8):

From pypy.commits at gmail.com  Sun Aug  5 03:42:48 2018
From: pypy.commits at gmail.com (arigo)
Date: Sun, 05 Aug 2018 00:42:48 -0700 (PDT)
Subject: [pypy-commit] cffi default: Add test from the recent ctypes fix
 bpo-29565 (may fail on win64)
Message-ID: <5b66aa78.1c69fb81.ea3ba.22b2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3133:7b305f61d0e0
Date: 2018-08-05 09:42 +0200
http://bitbucket.org/cffi/cffi/changeset/7b305f61d0e0/

Log:	Add test from the recent ctypes fix bpo-29565 (may fail on win64)

diff --git a/testing/cffi0/backend_tests.py b/testing/cffi0/backend_tests.py
--- a/testing/cffi0/backend_tests.py
+++ b/testing/cffi0/backend_tests.py
@@ -1945,3 +1945,30 @@
         # only works with the Python FFI instances
         ffi = FFI(backend=self.Backend())
         assert ffi.sizeof("struct{int a;}") == ffi.sizeof("int")
+
+    def test_callback_large_struct(self):
+        ffi = FFI(backend=self.Backend())
+        # more than 8 bytes
+        ffi.cdef("struct foo_s { unsigned long a, b, c; };")
+        #
+        @ffi.callback("void(struct foo_s)")
+        def cb(s):
+            seen.append(ffi.typeof(s))
+            s.a += 1
+            s.b += 2
+            s.c += 3
+            seen.append(s.a)
+            seen.append(s.b)
+            seen.append(s.c)
+        #
+        s1 = ffi.new("struct foo_s *", {'a': 100, 'b': 200, 'c': 300})
+        seen = []
+        cb(s1[0])
+        assert len(seen) == 4
+        assert s1.a == 100     # unmodified
+        assert s1.b == 200
+        assert s1.c == 300
+        assert seen[0] == ffi.typeof("struct foo_s")
+        assert seen[1] == 101
+        assert seen[2] == 202
+        assert seen[3] == 303

From pypy.commits at gmail.com  Sun Aug  5 09:34:58 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 05 Aug 2018 06:34:58 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: unicode/utf8 translation fixes
Message-ID: <5b66fd02.1c69fb81.efa59.3a68@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94945:9c9e17b77c17
Date: 2018-08-04 23:51 -0700
http://bitbucket.org/pypy/pypy/changeset/9c9e17b77c17/

Log:	unicode/utf8 translation fixes

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -83,7 +83,7 @@
             ##    con.interact()
             except OperationError as e:
                 debug("OperationError:")
-                debug(" operror-type: " + e.w_type.getname(space).encode('utf-8'))
+                debug(" operror-type: " + e.w_type.getname(space))
                 debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
                 return 1
         finally:
@@ -91,7 +91,7 @@
                 space.finish()
             except OperationError as e:
                 debug("OperationError:")
-                debug(" operror-type: " + e.w_type.getname(space).encode('utf-8'))
+                debug(" operror-type: " + e.w_type.getname(space))
                 debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
                 return 1
         return exitcode
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -515,7 +515,7 @@
                     elif fmt == 'T':
                         result = _decode_utf8(space.type(value).name)
                     elif fmt == 'N':
-                        result = value.getname(space)
+                        result = _decode_utf8(value.getname(space))
                     elif fmt == '8':
                         if isinstance(value, unicode):
                             result = value.encode('utf8')
diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py
--- a/pypy/interpreter/mixedmodule.py
+++ b/pypy/interpreter/mixedmodule.py
@@ -130,7 +130,7 @@
                     bltin.w_module = self.w_name
                     func._builtinversion_ = bltin
                     bltin.name = name
-                    bltin.qualname = bltin.name.decode('utf-8')
+                    bltin.qualname = bltin.name
                 w_value = bltin
             space.setitem(self.w_dict, w_name, w_value)
             return w_value
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -74,6 +74,10 @@
                 break
         return i
 
+    @specialize.arg(1)
+    def _raise(self, msg, *args):
+        raise oefmt(self.space.w_ValueError, msg, *args)
+
     def decode_any(self, i):
         i = self.skip_whitespace(i)
         ch = self.ll_chars[i]
diff --git a/pypy/module/_pypyjson/interp_encoder.py b/pypy/module/_pypyjson/interp_encoder.py
--- a/pypy/module/_pypyjson/interp_encoder.py
+++ b/pypy/module/_pypyjson/interp_encoder.py
@@ -1,7 +1,4 @@
 from rpython.rlib.rstring import StringBuilder
-from rpython.rlib import rutf8
-from pypy.interpreter import unicodehelper
-
 
 HEX = '0123456789abcdef'
 
@@ -17,7 +14,7 @@
 
 
 def raw_encode_basestring_ascii(space, w_unicode):
-    u = space.utf8_w(w_unicode).encode()
+    u = space.utf8_w(w_unicode)
     for i in range(len(u)):
         c = ord(u[i])
         if c < 32 or c > 126 or c == ord('\\') or c == ord('"'):
diff --git a/pypy/module/cpyext/classobject.py b/pypy/module/cpyext/classobject.py
--- a/pypy/module/cpyext/classobject.py
+++ b/pypy/module/cpyext/classobject.py
@@ -38,7 +38,7 @@
 
     def descr_repr(self, space):
         return self.getrepr(space, u'<instancemethod %s>' %
-                            (self.w_function.getname(space),))
+                            (self.w_function.getname(space).decode('utf8'),))
 
 InstanceMethod.typedef = TypeDef("instancemethod",
     __new__ = interp2app(InstanceMethod.descr_new),
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -72,7 +72,7 @@
 
 def unicode_attach(space, py_obj, w_obj, w_userdata=None):
     "Fills a newly allocated PyUnicodeObject with a unicode string"
-    value = space.utf8_w(w_obj).decode()
+    value = space.utf8_w(w_obj).decode('utf8')
     set_wsize(py_obj, len(value))
     set_wbuffer(py_obj, lltype.nullptr(rffi.CWCHARP.TO))
     _readify(space, py_obj, value)
@@ -353,7 +353,7 @@
     if not get_wbuffer(ref):
         # Copy unicode buffer
         w_unicode = from_ref(space, rffi.cast(PyObject, ref))
-        u = space.utf8_w(w_unicode).decode()
+        u = space.utf8_w(w_unicode).decode('utf8')
         set_wbuffer(ref, rffi.unicode2wcharp(u))
         set_wsize(ref, len(u))
     if psize:
@@ -943,7 +943,7 @@
     than, equal, and greater than, respectively. It is best to pass only
     ASCII-encoded strings, but the function interprets the input string as
     ISO-8859-1 if it contains non-ASCII characters."""
-    uni = space.utf8_w(w_uni).decode()
+    uni = space.utf8_w(w_uni).decode('utf8')
     i = 0
     # Compare Unicode string and source character set string
     while i < len(uni) and string[i] != '\0':
@@ -1054,7 +1054,7 @@
 
 @cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject)
 def PyUnicode_Substring(space, w_str, start, end):
-    usrc = space.utf8_w(w_str).decode()
+    usrc = space.utf8_w(w_str).decode('utf8')
     length = len(usrc)
     if start < 0 or end < 0:
         raise oefmt(space.w_IndexError, "string index out of range")
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -753,7 +753,7 @@
     def getfulltypename(self, w_obj):
         w_type = self.type(w_obj)
         if w_type.is_heaptype():
-            classname = w_type.getqualname(self)
+            classname = w_type.getqualname(self).decode('utf8')
             w_module = w_type.lookup("__module__")
             if w_module is not None:
                 try:
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -579,7 +579,7 @@
                 result = self.name[dot+1:]
             else:
                 result = self.name
-        return result.decode('utf-8')
+        return result
 
     def getqualname(self, space):
         return self.qualname
@@ -792,7 +792,6 @@
         w_typetype = w_winner
 
     name = space.text_w(w_name) # NB. CPython forbids surrogates here
-    assert isinstance(name, str)
     if '\x00' in name:
         raise oefmt(space.w_ValueError, "type name must not contain null characters")
     dict_w = {}
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -377,6 +377,8 @@
                 codes = unicodedb.tolower_full(ch)
             elif unicodedb.islower(ch):
                 codes = unicodedb.toupper_full(ch)
+            else:
+                codes = [ch,]
             for c in codes:
                 builder.append_code(c)
         return self.from_utf8builder(builder)
diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py
--- a/rpython/rlib/rutf8.py
+++ b/rpython/rlib/rutf8.py
@@ -28,7 +28,7 @@
 
 
 # we need a way to accept both r_uint and int(nonneg=True)
-#@signature(types.int_nonneg(), types.bool(), returns=types.str())
+ at signature(types.int_nonneg(), types.bool(), returns=types.str())
 def unichr_as_utf8(code, allow_surrogates=False):
     """Encode code (numeric value) as utf8 encoded string
     """

From pypy.commits at gmail.com  Sun Aug  5 09:35:00 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 05 Aug 2018 06:35:00 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: remove str_w
Message-ID: <5b66fd04.1c69fb81.ba9db.37db@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94946:71093bf7b39c
Date: 2018-08-04 23:51 -0700
http://bitbucket.org/pypy/pypy/changeset/71093bf7b39c/

Log:	remove str_w

diff --git a/pypy/module/_locale/test/test_locale.py b/pypy/module/_locale/test/test_locale.py
--- a/pypy/module/_locale/test/test_locale.py
+++ b/pypy/module/_locale/test/test_locale.py
@@ -25,21 +25,21 @@
                 # some systems are only UTF-8 oriented
                 try:
                     _locale.setlocale(_locale.LC_ALL,
-                                      cls.space.str_w(cls.w_language_en))
+                                      cls.space.utf8_w(cls.w_language_en))
                 except _locale.Error:
                     _locale.setlocale(_locale.LC_ALL,
-                                      cls.space.str_w(cls.w_language_utf8))
+                                      cls.space.utf8_w(cls.w_language_utf8))
                     cls.w_language_en = cls.w_language_utf8
 
                 _locale.setlocale(_locale.LC_ALL,
-                                  cls.space.str_w(cls.w_language_pl))
+                                  cls.space.utf8_w(cls.w_language_pl))
             except _locale.Error:
                 py.test.skip("necessary locales not installed")
 
             # Windows forbids the UTF-8 character set since Windows XP.
             try:
                 _locale.setlocale(_locale.LC_ALL,
-                                  cls.space.str_w(cls.w_language_utf8))
+                                  cls.space.utf8_w(cls.w_language_utf8))
             except _locale.Error:
                 del cls.w_language_utf8
         finally:

From pypy.commits at gmail.com  Sun Aug  5 09:35:02 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 05 Aug 2018 06:35:02 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: translation fixes,
 space.newtext() should not accept unicode
Message-ID: <5b66fd06.1c69fb81.9696e.4157@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94947:73ecf23722a4
Date: 2018-08-05 06:34 -0700
http://bitbucket.org/pypy/pypy/changeset/73ecf23722a4/

Log:	translation fixes, space.newtext() should not accept unicode

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -83,7 +83,7 @@
             return space.utf8_w(space.getattr(self, space.newtext('__name__')))
         except OperationError as e:
             if e.match(space, space.w_TypeError) or e.match(space, space.w_AttributeError):
-                return u'?'
+                return '?'
             raise
 
     def getaddrstring(self, space):
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -520,7 +520,7 @@
                         if isinstance(value, unicode):
                             result = value.encode('utf8')
                         else:
-                            result = value.decode('utf8', errors='replace')
+                            result = value.decode('utf8', 'replace')
                     else:
                         if isinstance(value, unicode):
                             result = value
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -215,7 +215,7 @@
             e2.record_context(space, space.getexecutioncontext())
             raise e2
         else:
-            space.warn(space.newtext(u"generator '%s' raised StopIteration"
+            space.warn(space.newtext("generator '%s' raised StopIteration"
                                         % self.get_qualname()),
                        space.w_PendingDeprecationWarning)
 
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -1587,7 +1587,7 @@
                                   errorhandler=errorhandler)
     size = len(s)
     if size == 0:
-        return '', 0
+        return '', 0, 0
 
     pos = 0
     result = StringBuilder(size)
diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py
--- a/pypy/module/__builtin__/descriptor.py
+++ b/pypy/module/__builtin__/descriptor.py
@@ -30,14 +30,14 @@
 
     def descr_repr(self, space):
         if self.w_objtype is not None:
-            objtype_name = u"<%s object>" % self.w_objtype.getname(space)
+            objtype_name = "<%s object>" % self.w_objtype.getname(space)
         else:
-            objtype_name = u'NULL'
+            objtype_name = 'NULL'
         if self.w_starttype is not None:
             starttype_name = self.w_starttype.getname(space)
         else:
-            starttype_name = u'NULL'
-        return space.newtext(u"<super: <class '%s'>, %s>" % (
+            starttype_name = 'NULL'
+        return space.newtext("<super: <class '%s'>, %s>" % (
             starttype_name, objtype_name))
 
     def get(self, space, w_obj, w_type=None):
diff --git a/pypy/module/_codecs/locale.py b/pypy/module/_codecs/locale.py
--- a/pypy/module/_codecs/locale.py
+++ b/pypy/module/_codecs/locale.py
@@ -57,7 +57,7 @@
                 if errorpos == -1:
                     raise MemoryError
                 errmsg = _errmsg("pypy_wchar2char")
-                errorhandler('strict', 'filesystemencoding', errmsg, u,
+                errorhandler('strict', 'filesystemencoding', errmsg, u.encode('utf8'),
                              errorpos, errorpos + 1)
             return rffi.charp2str(sbuf)
         finally:
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -85,7 +85,7 @@
     s = rffi.wcharpsize2unicode(get_wbuffer(py_obj), get_wsize(py_obj))
     w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type))
     w_obj = space.allocate_instance(unicodeobject.W_UnicodeObject, w_type)
-    w_obj.__init__(s)
+    w_obj.__init__(s, len(s))
     track_reference(space, py_obj, w_obj)
     return w_obj
 
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -381,8 +381,9 @@
 
     @specialize.argtype(1)
     def newtext(self, s):
-        if isinstance(s, unicode):
-            s, lgt = s.encode('utf8'), len(s)
+        assert not isinstance(s, unicode)
+        #if isinstance(s, unicode):
+            #s, lgt = s.encode('utf8'), len(s)
         elif isinstance(s, str):
             s, lgt, codepoints = decode_utf8sp(self, s)
         elif isinstance(s, tuple):
@@ -391,6 +392,7 @@
         else:
             # XXX what is s ?
             lgt = rutf8.check_utf8(s, True)
+        assert isinstance(s, str)
         return W_UnicodeObject(s, lgt)
 
     def newtext_or_none(self, s):
@@ -399,6 +401,7 @@
         return self.newtext(s)
 
     def newutf8(self, utf8s, length):
+        assert length >= 0
         assert isinstance(utf8s, str)
         return W_UnicodeObject(utf8s, length)
 

From pypy.commits at gmail.com  Sun Aug  5 14:13:20 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 05 Aug 2018 11:13:20 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: whoops
Message-ID: <5b673e40.1c69fb81.e66ef.0d90@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94948:5918348069b9
Date: 2018-08-05 06:50 -0700
http://bitbucket.org/pypy/pypy/changeset/5918348069b9/

Log:	whoops

diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -384,7 +384,7 @@
         assert not isinstance(s, unicode)
         #if isinstance(s, unicode):
             #s, lgt = s.encode('utf8'), len(s)
-        elif isinstance(s, str):
+        if isinstance(s, str):
             s, lgt, codepoints = decode_utf8sp(self, s)
         elif isinstance(s, tuple):
             # result of decode_utf8

From pypy.commits at gmail.com  Sun Aug  5 14:13:22 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 05 Aug 2018 11:13:22 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: w_type.getname returns utf8
Message-ID: <5b673e42.1c69fb81.36058.fb74@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94949:da2e74e209e0
Date: 2018-08-05 10:34 -0700
http://bitbucket.org/pypy/pypy/changeset/da2e74e209e0/

Log:	w_type.getname returns utf8

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -148,7 +148,7 @@
         except OperationError as e:
             if verbose:
                 debug("OperationError:")
-                debug(" operror-type: " + e.w_type.getname(space).encode('utf-8'))
+                debug(" operror-type: " + e.w_type.getname(space))
                 debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
             return rffi.cast(rffi.INT, -1)
         finally:
@@ -202,7 +202,7 @@
             """)
         except OperationError as e:
             debug("OperationError:")
-            debug(" operror-type: " + e.w_type.getname(space).encode('utf-8'))
+            debug(" operror-type: " + e.w_type.getname(space))
             debug(" operror-value: " + space.text_w(space.str(e.get_w_value(space))))
             return -1
         return 0
diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py
--- a/pypy/interpreter/test/test_executioncontext.py
+++ b/pypy/interpreter/test/test_executioncontext.py
@@ -178,7 +178,7 @@
                 w_class = space.type(seen[0].w_instance)
                 found = 'method %s of %s' % (
                     seen[0].w_function.name,
-                    w_class.getname(space).encode('utf-8'))
+                    w_class.getname(space))
             else:
                 assert isinstance(seen[0], Function)
                 found = 'builtin %s' % seen[0].name
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -88,7 +88,7 @@
         py_getsetdef.c_doc = rffi.str2charp(doc)
     else:
         py_getsetdef.c_doc = rffi.cast(rffi.CCHARP, 0)
-    py_getsetdef.c_name = rffi.str2charp(getsetprop.getname(space).encode('utf-8'))
+    py_getsetdef.c_name = rffi.str2charp(getsetprop.getname(space))
     # XXX FIXME - actually assign these !!!
     py_getsetdef.c_get = cts.cast('getter', 0)
     py_getsetdef.c_set = cts.cast('setter', 0)
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -762,7 +762,7 @@
 
 def _get_printable_location(w_type):
     return ('bytearray_from_byte_sequence [w_type=%s]' %
-            w_type.getname(w_type.space).encode('utf-8'))
+            w_type.getname(w_type.space))
 
 _byteseq_jitdriver = jit.JitDriver(
     name='bytearray_from_byte_sequence',
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -132,7 +132,7 @@
 
 def _get_printable_location(w_type):
     return ('list__do_extend_from_iterable [w_type=%s]' %
-            w_type.getname(w_type.space).encode('utf-8'))
+            w_type.getname(w_type.space))
 
 
 _do_extend_jitdriver = jit.JitDriver(
diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py
--- a/pypy/tool/pytest/appsupport.py
+++ b/pypy/tool/pytest/appsupport.py
@@ -97,7 +97,7 @@
     def __init__(self, space, operr):
         self.space = space
         self.operr = operr
-        self.typename = operr.w_type.getname(space).encode('utf-8')
+        self.typename = operr.w_type.getname(space)
         self.traceback = AppTraceback(space, self.operr.get_traceback())
         debug_excs = getattr(operr, 'debug_excs', [])
         if debug_excs:

From pypy.commits at gmail.com  Sun Aug  5 14:13:24 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 05 Aug 2018 11:13:24 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: use a signature to find where
 len < 0, fix
Message-ID: <5b673e44.1c69fb81.372b5.975b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94950:9c9fca815956
Date: 2018-08-05 10:36 -0700
http://bitbucket.org/pypy/pypy/changeset/9c9fca815956/

Log:	use a signature to find where len < 0, fix

diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -307,6 +307,7 @@
             # ascii only, fast path (ascii is a strict subset of
             # latin1, and we already checked that all the chars are <
             # 128)
+            assert end >= start
             return self.space.newutf8(self.getslice(start, end),
                                       end - start)
 
diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py
--- a/pypy/module/cpyext/longobject.py
+++ b/pypy/module/cpyext/longobject.py
@@ -200,6 +200,8 @@
     string, length gives the number of characters, and base is the radix
     for the conversion.  The radix must be in the range [2, 36]; if it is
     out of range, ValueError will be raised."""
+    if length < 0:
+        length = 0
     w_value = space.newutf8(wcharpsize2utf8(space, u, length), length)
     return PyLong_FromUnicodeObject(space, w_value, base)
 
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -472,6 +472,8 @@
     object. If the buffer is not NULL, the return value might be a shared object.
     Therefore, modification of the resulting Unicode object is only allowed when u
     is NULL."""
+    if length < 0:
+        length = 0
     if wchar_p:
         s = wcharpsize2utf8(space, wchar_p, length)
         return make_ref(space, space.newutf8(s, length))
@@ -755,6 +757,8 @@
         """Encode the Py_UNICODE buffer of the given size and return a
         Python string object.  Return NULL if an exception was raised
         by the codec."""
+        if size < 0:
+            size = 0
         u = wcharpsize2utf8(space, s, size)
         w_u = space.newutf8(u, size)
         if errors:
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -13,7 +13,8 @@
 from rpython.rlib.rarithmetic import base_int, widen, is_valid_int
 from rpython.rlib.objectmodel import import_from_mixin, we_are_translated
 from rpython.rlib.objectmodel import not_rpython
-from rpython.rlib import jit, rutf8
+from rpython.rlib import jit, rutf8, types
+from rpython.rlib.signature import signature, finishsigs
 
 # Object imports
 from pypy.objspace.std.boolobject import W_BoolObject
@@ -36,7 +37,7 @@
 from pypy.objspace.std.typeobject import W_TypeObject, TypeCache
 from pypy.objspace.std.unicodeobject import W_UnicodeObject
 
-
+ at finishsigs
 class StdObjSpace(ObjSpace):
     """The standard object space, implementing a general-purpose object
     library in Restricted Python."""
@@ -381,10 +382,9 @@
 
     @specialize.argtype(1)
     def newtext(self, s):
-        assert not isinstance(s, unicode)
-        #if isinstance(s, unicode):
-            #s, lgt = s.encode('utf8'), len(s)
-        if isinstance(s, str):
+        if isinstance(s, unicode):
+            s, lgt = s.encode('utf8'), len(s)
+        elif isinstance(s, str):
             s, lgt, codepoints = decode_utf8sp(self, s)
         elif isinstance(s, tuple):
             # result of decode_utf8
@@ -400,8 +400,9 @@
             return self.w_None
         return self.newtext(s)
 
+    # XXX find where length is annotated as negative int
+    @signature(types.any(), types.str(), types.int_nonneg(), returns=types.any())
     def newutf8(self, utf8s, length):
-        assert length >= 0
         assert isinstance(utf8s, str)
         return W_UnicodeObject(utf8s, length)
 
diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py
--- a/rpython/rlib/rutf8.py
+++ b/rpython/rlib/rutf8.py
@@ -28,7 +28,7 @@
 
 
 # we need a way to accept both r_uint and int(nonneg=True)
- at signature(types.int_nonneg(), types.bool(), returns=types.str())
+#@signature(types.int_nonneg(), types.bool(), returns=types.str())
 def unichr_as_utf8(code, allow_surrogates=False):
     """Encode code (numeric value) as utf8 encoded string
     """
diff --git a/rpython/rlib/signature.py b/rpython/rlib/signature.py
--- a/rpython/rlib/signature.py
+++ b/rpython/rlib/signature.py
@@ -9,7 +9,7 @@
       def foo(...)
 
     The arguments paramNtype and returntype should be instances
-    of the classes in rpython.annotator.types.
+    of the classes in rpython.rlib.types.
     """
     returntype = kwargs.pop('returns', None)
     if returntype is None:

From pypy.commits at gmail.com  Sun Aug  5 14:13:27 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 05 Aug 2018 11:13:27 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: use rutf8 to do decode(...,
 'replace', ...)
Message-ID: <5b673e47.1c69fb81.37f8c.8ee6@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94951:91d7f28bf01c
Date: 2018-08-05 11:08 -0700
http://bitbucket.org/pypy/pypy/changeset/91d7f28bf01c/

Log:	use rutf8 to do decode(..., 'replace', ...)

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -517,10 +517,14 @@
                     elif fmt == 'N':
                         result = _decode_utf8(value.getname(space))
                     elif fmt == '8':
+                        # u'str\uxxxx' -> 'str\xXX\xXX' -> u"'str\xXX\xXX'"
                         if isinstance(value, unicode):
                             result = value.encode('utf8')
                         else:
-                            result = value.decode('utf8', 'replace')
+                            from pypy.interpreter import unicodehelper
+                            result = _decode_utf8(unicodehelper.str_decode_utf8(
+                                value, 'replace', True,
+                                unicodehelper.decode_never_raise, True)[0])
                     else:
                         if isinstance(value, unicode):
                             result = value

From pypy.commits at gmail.com  Sun Aug  5 14:13:29 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 05 Aug 2018 11:13:29 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: translation fixes
Message-ID: <5b673e49.1c69fb81.b4ac1.50be@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94952:ff1a45089342
Date: 2018-08-05 11:09 -0700
http://bitbucket.org/pypy/pypy/changeset/ff1a45089342/

Log:	translation fixes

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -39,8 +39,7 @@
     # Fast version of the "strict" errors handler.
     def raise_unicode_exception_encode(errors, encoding, msg, utf8,
                                        startingpos, endingpos):
-        if isinstance(utf8, unicode):
-            utf8 = utf8.encode('utf8')
+        assert not isinstance(utf8, unicode)
         u_len = rutf8.get_utf8_length(utf8)
         raise OperationError(space.w_UnicodeEncodeError,
                              space.newtuple([space.newtext(encoding),
@@ -362,7 +361,7 @@
                 if not final:
                     pos -= 1
                     break
-                r, pos = errorhandler(errors, "utf8", "unexpected end of data",
+                r, pos, lgt = errorhandler(errors, "utf8", "unexpected end of data",
                     s, pos - 1, pos + 1)
                 res.append(r)
                 continue
diff --git a/pypy/module/_weakref/interp__weakref.py b/pypy/module/_weakref/interp__weakref.py
--- a/pypy/module/_weakref/interp__weakref.py
+++ b/pypy/module/_weakref/interp__weakref.py
@@ -182,10 +182,10 @@
         else:
             typename = space.type(w_obj).getname(space)
             objname = w_obj.getname(space)
-            if objname and objname != u'?':
-                state = u"; to '%s' (%s)" % (typename, objname)
+            if objname and objname != '?':
+                state = "; to '%s' (%s)" % (typename, objname)
             else:
-                state = u"; to '%s'" % (typename,)
+                state = "; to '%s'" % (typename,)
         return self.getrepr(space, unicode(self.typedef.name), state)
 
 
diff --git a/pypy/module/unicodedata/interp_ucd.py b/pypy/module/unicodedata/interp_ucd.py
--- a/pypy/module/unicodedata/interp_ucd.py
+++ b/pypy/module/unicodedata/interp_ucd.py
@@ -6,7 +6,7 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.typedef import TypeDef, interp_attrproperty
-from rpython.rlib.rarithmetic import r_longlong
+from rpython.rlib.rarithmetic import r_longlong, r_uint
 from rpython.rlib.unicodedata import unicodedb_8_0_0, unicodedb_3_2_0
 from rpython.rlib.rutf8 import Utf8StringBuilder, unichr_as_utf8
 
@@ -82,7 +82,7 @@
         sequence = self._lookup_named_sequence(code)
         if sequence is not None:
             # named sequences only contain UCS2 codes, no surrogates &co.
-            return space.newutf8(unichr_as_utf8(code), 1)
+            return space.newutf8(unichr_as_utf8(r_uint(code)), 1)
 
 
 
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -1438,7 +1438,7 @@
         typename = space.type(self).getname(space)
         w_seq = space.call_function(space.w_list, self)
         seq_repr = space.utf8_w(space.repr(w_seq))
-        return space.newtext(u"%s(%s)" % (typename, seq_repr.decode('utf8')))
+        return space.newtext("%s(%s)" % (typename, seq_repr))
 
     def descr_len(self, space):
         return space.len(self.w_dict)

From pypy.commits at gmail.com  Sun Aug  5 15:17:46 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 05 Aug 2018 12:17:46 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: try making space.newtext
 accept only utf8
Message-ID: <5b674d5a.1c69fb81.8bb11.3e7c@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94953:1b6dfea3eef5
Date: 2018-08-05 12:16 -0700
http://bitbucket.org/pypy/pypy/changeset/1b6dfea3eef5/

Log:	try making space.newtext accept only utf8

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -105,9 +105,9 @@
             w_id = space.rshift(w_id, w_4)
         return ''.join(addrstring)
 
-    def getrepr(self, space, info, moreinfo=u''):
-        addrstring = unicode(self.getaddrstring(space))
-        return space.newtext(u"<%s at 0x%s%s>" % (info, addrstring, moreinfo))
+    def getrepr(self, space, info, moreinfo=''):
+        addrstring = self.getaddrstring(space)
+        return space.newtext("<%s at 0x%s%s>" % (info, addrstring, moreinfo))
 
     def getslotvalue(self, index):
         raise NotImplementedError
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -42,10 +42,8 @@
 
     def descr__repr__(self, space):
         addrstring = self.getaddrstring(space)
-        return space.newtext(u"<%s object %s at 0x%s>" %
-                          (unicode(self.KIND),
-                           self.get_qualname(),
-                           unicode(addrstring)))
+        return space.newtext("<%s object %s at 0x%s>" %
+                          (self.KIND, self.get_qualname(), addrstring))
 
     def descr_send(self, w_arg):
         """send(arg) -> send 'arg' into generator/coroutine,
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1628,7 +1628,7 @@
         if (oparg & consts.FVS_MASK) == consts.FVS_HAVE_SPEC:
             w_spec = self.popvalue()
         else:
-            w_spec = space.newtext(u'')
+            w_spec = space.newtext('')
         w_value = self.popvalue()
         #
         conversion = oparg & consts.FVC_MASK
@@ -1649,9 +1649,9 @@
         lst = []
         for i in range(itemcount-1, -1, -1):
             w_item = self.peekvalue(i)
-            lst.append(space.realunicode_w(w_item))
+            lst.append(space.utf8_w(w_item))
         self.dropvalues(itemcount)
-        w_res = space.newtext(u''.join(lst))
+        w_res = space.newtext(''.join(lst))
         self.pushvalue(w_res)
 
     def _revdb_load_var(self, oparg):
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -74,8 +74,8 @@
                               force_ignore=False)[0]
     elif _MACOSX:
         bytes = space.bytes_w(w_string)
-        uni = runicode.str_decode_utf_8_impl(
-            bytes, len(bytes), 'surrogateescape', final=True,
+        uni = str_decode_utf8(
+            bytes, 'surrogateescape', final=True,
             errorhandler=state.decode_error_handler,
             allow_surrogates=False)[0]
     elif space.sys.filesystemencoding is None or state.codec_need_encodings:
@@ -296,15 +296,13 @@
 
 if sys.platform == 'win32':
     def utf8_encode_mbcs(s, slen, errors, errorhandler):
-        from rpython.rlib import runicode
         s = s.decode('utf-8')
-        res = runicode.unicode_encode_mbcs(s, slen, errors, errorhandler)
+        res = unicode_encode_mbcs(s, slen, errors, errorhandler)
         return res
         
     def str_decode_mbcs(s, errors, final, errorhandler):
-        from rpython.rlib import runicode
         slen = len(s)
-        res, size = runicode.str_decode_mbcs(s, slen, final=final, errors=errors,
+        res, size = str_decode_mbcs(s, slen, final=final, errors=errors,
                                            errorhandler=errorhandler)
         return res.encode('utf8'), len(res)
 
diff --git a/pypy/module/__pypy__/interp_stderrprinter.py b/pypy/module/__pypy__/interp_stderrprinter.py
--- a/pypy/module/__pypy__/interp_stderrprinter.py
+++ b/pypy/module/__pypy__/interp_stderrprinter.py
@@ -16,8 +16,8 @@
         self.fd = fd
 
     def descr_repr(self, space):
-        addrstring = unicode(self.getaddrstring(space))
-        return space.newtext(u"<StdErrPrinter(fd=%d) object at 0x%s>" %
+        addrstring = self.getaddrstring(space)
+        return space.newtext("<StdErrPrinter(fd=%d) object at 0x%s>" %
                                 (self.fd, addrstring))
 
     def descr_noop(self, space):
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -3,7 +3,7 @@
 from rpython.rlib.objectmodel import we_are_translated, not_rpython
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib import runicode
-from rpython.rlib.runicode import ( raw_unicode_escape_helper_unicode)
+from rpython.rlib.runicode import raw_unicode_escape_helper_unicode
 from rpython.rlib import rutf8
 
 from pypy.interpreter.error import OperationError, oefmt
diff --git a/pypy/module/_io/interp_stringio.py b/pypy/module/_io/interp_stringio.py
--- a/pypy/module/_io/interp_stringio.py
+++ b/pypy/module/_io/interp_stringio.py
@@ -219,11 +219,13 @@
                 self.w_decoder, "decode", w_obj, space.w_True)
         else:
             w_decoded = w_obj
-        if self.writenl:
+        writenl = self.writenl
+        if writenl is not None:
             w_decoded = space.call_method(
                 w_decoded, "replace",
-                space.newtext("\n"), space.newutf8(self.writenl,
-                    get_utf8_length(self.writenl)))
+                space.newtext("\n"),
+                space.newutf8(writenl, get_utf8_length(writenl)),
+            )
         string = space.utf8_w(w_decoded)
         if string:
             self.buf.write(string)
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -1,7 +1,7 @@
 import sys
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.objectmodel import specialize, always_inline, r_dict
-from rpython.rlib import rfloat, runicode, rutf8
+from rpython.rlib import rfloat, rutf8
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.interpreter.error import oefmt, OperationError
 from rpython.rlib.rarithmetic import r_uint
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -451,7 +451,7 @@
                 return space.newbytes(func(add_arg, argdesc, ll_type))
             elif c == 'u':
                 return space.newutf8(rutf8.unichr_as_utf8(
-                    ord(func(add_arg, argdesc, ll_type))), 1)
+                    r_uint(ord(func(add_arg, argdesc, ll_type)))), 1)
             elif c == 'f' or c == 'd' or c == 'g':
                 return space.newfloat(float(func(add_arg, argdesc, ll_type)))
             else:
@@ -615,6 +615,8 @@
 def wcharp2rawunicode(space, address, maxlength=-1):
     if maxlength == -1:
         return wcharp2unicode(space, address)
+    elif maxlength < 0:
+        maxlength = 0
     s = rffi.wcharpsize2utf8(rffi.cast(rffi.CWCHARP, address), maxlength)
     return space.newutf8(s, maxlength)
 
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -123,12 +123,12 @@
         if flags != 0:
             flag_items.append('0x%x' % flags)
         if len(flag_items) == 0:
-            usep = u''
-            uflags = u''
+            usep = ''
+            uflags = ''
         else:
-            usep = u', '
-            uflags = u'|'.join([item.decode('latin-1') for item in flag_items])
-        return space.newtext(u're.compile(%s%s%s)' % (u, usep, uflags))
+            usep = ', '
+            uflags = '|'.join(flag_items)
+        return space.newtext('re.compile(%s%s%s)' % (u, usep, uflags))
 
     def fget_groupindex(self, space):
         w_groupindex = self.w_groupindex
@@ -424,7 +424,7 @@
                 return space.newtext(unicodebuilder.build()), n
         else:
             if space.isinstance_w(w_string, space.w_unicode):
-                w_emptystr = space.newtext(u'')
+                w_emptystr = space.newtext('')
             else:
                 w_emptystr = space.newbytes('')
             w_item = space.call_method(w_emptystr, 'join',
@@ -528,10 +528,10 @@
         ctx = self.ctx
         start, end = ctx.match_start, ctx.match_end
         w_s = slice_w(space, ctx, start, end, space.w_None)
-        u = space.realunicode_w(space.repr(w_s))
+        u = space.utf8_w(space.repr(w_s))
         if len(u) > 50:
             u = u[:50]
-        return space.newtext(u'<_sre.SRE_Match object; span=(%d, %d), match=%s>' %
+        return space.newtext('<_sre.SRE_Match object; span=(%d, %d), match=%s>' %
                           (start, end, u))
 
     def cannot_copy_w(self):
diff --git a/pypy/module/_weakref/interp__weakref.py b/pypy/module/_weakref/interp__weakref.py
--- a/pypy/module/_weakref/interp__weakref.py
+++ b/pypy/module/_weakref/interp__weakref.py
@@ -178,7 +178,7 @@
     def descr__repr__(self, space):
         w_obj = self.dereference()
         if w_obj is None:
-            state = u'; dead'
+            state = '; dead'
         else:
             typename = space.type(w_obj).getname(space)
             objname = w_obj.getname(space)
@@ -186,7 +186,7 @@
                 state = "; to '%s' (%s)" % (typename, objname)
             else:
                 state = "; to '%s'" % (typename,)
-        return self.getrepr(space, unicode(self.typedef.name), state)
+        return self.getrepr(space, self.typedef.name, state)
 
 
 class W_Weakref(W_WeakrefBase):
diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py
--- a/pypy/module/_winreg/interp_winreg.py
+++ b/pypy/module/_winreg/interp_winreg.py
@@ -33,7 +33,7 @@
         return space.newint(self.as_int())
 
     def descr_repr(self, space):
-        return space.newtext(u"<PyHKEY:0x%x>" % (self.as_int(),))
+        return space.newtext("<PyHKEY:0x%x>" % (self.as_int(),))
 
     def descr_int(self, space):
         return space.newint(self.as_int())
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -935,7 +935,7 @@
   the file descriptor must refer to a directory.
   If this functionality is unavailable, using it raises NotImplementedError."""
     if space.is_none(w_path):
-        w_path = space.newtext(u".")
+        w_path = space.newtext(".")
     if space.isinstance_w(w_path, space.w_bytes):
         # XXX CPython doesn't follow this path either if w_path is,
         # for example, a memoryview or another buffer type
diff --git a/pypy/module/posix/interp_scandir.py b/pypy/module/posix/interp_scandir.py
--- a/pypy/module/posix/interp_scandir.py
+++ b/pypy/module/posix/interp_scandir.py
@@ -14,7 +14,7 @@
 def scandir(space, w_path=None):
     "scandir(path='.') -> iterator of DirEntry objects for given path"
     if space.is_none(w_path):
-        w_path = space.newtext(u".")
+        w_path = space.newtext(".")
 
     if not _WIN32:
         if space.isinstance_w(w_path, space.w_bytes):
diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -5,7 +5,7 @@
 import string
 
 from pypy.interpreter.error import OperationError, oefmt
-from rpython.rlib import rstring, runicode, rlocale, rfloat, jit, rutf8
+from rpython.rlib import rstring, rlocale, rfloat, jit, rutf8
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rfloat import formatd
 from rpython.rlib.rarithmetic import r_uint, intmask

From pypy.commits at gmail.com  Mon Aug  6 03:19:11 2018
From: pypy.commits at gmail.com (mattip)
Date: Mon, 06 Aug 2018 00:19:11 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: translation - len must be
 non-negative
Message-ID: <5b67f66f.1c69fb81.7734a.13ae@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94954:22f9db3e5c61
Date: 2018-08-05 14:52 -0700
http://bitbucket.org/pypy/pypy/changeset/22f9db3e5c61/

Log:	translation - len must be non-negative

diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -307,9 +307,9 @@
             # ascii only, fast path (ascii is a strict subset of
             # latin1, and we already checked that all the chars are <
             # 128)
-            assert end >= start
-            return self.space.newutf8(self.getslice(start, end),
-                                      end - start)
+            lgt = end - start
+            assert lgt >= 0
+            return self.space.newutf8(self.getslice(start, end), lgt)
 
     def decode_string_escaped(self, start):
         i = self.pos
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -494,8 +494,11 @@
         an array of some other type.
         """
         if self.typecode == 'u':
+            s = self.len
+            if s < 0:
+                s = 0
             buf = rffi.cast(UNICODE_ARRAY, self._buffer_as_unsigned())
-            return space.newutf8(rffi.wcharpsize2utf8(buf, self.len), self.len)
+            return space.newutf8(rffi.wcharpsize2utf8(buf, s), s)
         else:
             raise oefmt(space.w_ValueError,
                         "tounicode() may only be called on type 'u' arrays")
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -85,7 +85,7 @@
     s = rffi.wcharpsize2unicode(get_wbuffer(py_obj), get_wsize(py_obj))
     w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type))
     w_obj = space.allocate_instance(unicodeobject.W_UnicodeObject, w_type)
-    w_obj.__init__(s, len(s))
+    w_obj.__init__(s.encode('utf8'), len(s))
     track_reference(space, py_obj, w_obj)
     return w_obj
 
@@ -271,7 +271,7 @@
     assert isinstance(w_obj, unicodeobject.W_UnicodeObject)
     py_obj = as_pyobj(space, w_obj)
     assert get_kind(py_obj) == WCHAR_KIND
-    return _readify(space, py_obj, w_obj._value)
+    return _readify(space, py_obj, space.utf8_w(w_obj))
 
 def _readify(space, py_obj, value):
     maxchar = 0

From pypy.commits at gmail.com  Mon Aug  6 03:19:13 2018
From: pypy.commits at gmail.com (mattip)
Date: Mon, 06 Aug 2018 00:19:13 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: more internal unicode -> utf8
Message-ID: <5b67f671.1c69fb81.459d5.46e5@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94955:3814aec634e5
Date: 2018-08-05 14:53 -0700
http://bitbucket.org/pypy/pypy/changeset/3814aec634e5/

Log:	more internal unicode -> utf8

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -515,7 +515,11 @@
                     elif fmt == 'T':
                         result = _decode_utf8(space.type(value).name)
                     elif fmt == 'N':
-                        result = _decode_utf8(value.getname(space))
+                        name = value.getname(space)
+                        if isinstance(name, unicode):
+                            result = name
+                        else:
+                            result = _decode_utf8(name)
                     elif fmt == '8':
                         # u'str\uxxxx' -> 'str\xXX\xXX' -> u"'str\xXX\xXX'"
                         if isinstance(value, unicode):
diff --git a/pypy/module/cpyext/classobject.py b/pypy/module/cpyext/classobject.py
--- a/pypy/module/cpyext/classobject.py
+++ b/pypy/module/cpyext/classobject.py
@@ -37,8 +37,8 @@
         return space.call_args(self.w_function, __args__)
 
     def descr_repr(self, space):
-        return self.getrepr(space, u'<instancemethod %s>' %
-                            (self.w_function.getname(space).decode('utf8'),))
+        return self.getrepr(space, '<instancemethod %s>' %
+                            (self.w_function.getname(space),))
 
 InstanceMethod.typedef = TypeDef("instancemethod",
     __new__ = interp2app(InstanceMethod.descr_new),
diff --git a/pypy/objspace/std/objectobject.py b/pypy/objspace/std/objectobject.py
--- a/pypy/objspace/std/objectobject.py
+++ b/pypy/objspace/std/objectobject.py
@@ -165,7 +165,7 @@
 
 def descr__repr__(space, w_obj):
     classname = space.getfulltypename(w_obj)
-    return w_obj.getrepr(space, u'%s object' % (classname,))
+    return w_obj.getrepr(space, '%s object' % (classname,))
 
 
 def descr__str__(space, w_obj):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -757,7 +757,7 @@
     def getfulltypename(self, w_obj):
         w_type = self.type(w_obj)
         if w_type.is_heaptype():
-            classname = w_type.getqualname(self).decode('utf8')
+            classname = w_type.getqualname(self)
             w_module = w_type.lookup("__module__")
             if w_module is not None:
                 try:
@@ -766,7 +766,7 @@
                     if not e.match(self, self.w_TypeError):
                         raise
                 else:
-                    classname = u'%s.%s' % (modulename.decode('utf8'), classname)
+                    classname = '%s.%s' % (modulename, classname)
         else:
-            classname = w_type.name.decode('utf-8')
+            classname = w_type.name
         return classname

From pypy.commits at gmail.com  Mon Aug  6 03:19:15 2018
From: pypy.commits at gmail.com (mattip)
Date: Mon, 06 Aug 2018 00:19:15 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: more internal utf8/unicode
 confusion
Message-ID: <5b67f673.1c69fb81.58fcb.36d5@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94956:74555d57f729
Date: 2018-08-05 15:14 -0700
http://bitbucket.org/pypy/pypy/changeset/74555d57f729/

Log:	more internal utf8/unicode confusion

diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -256,7 +256,7 @@
         return self.call_args(__args__)
 
     def descr_function_repr(self):
-        return self.getrepr(self.space, u'function %s' % self.qualname.decode('utf8'))
+        return self.getrepr(self.space, 'function %s' % self.qualname)
 
 
     def _cleanup_(self):
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -271,7 +271,7 @@
     assert isinstance(w_obj, unicodeobject.W_UnicodeObject)
     py_obj = as_pyobj(space, w_obj)
     assert get_kind(py_obj) == WCHAR_KIND
-    return _readify(space, py_obj, space.utf8_w(w_obj))
+    return _readify(space, py_obj, space.utf8_w(w_obj).decode('utf8'))
 
 def _readify(space, py_obj, value):
     maxchar = 0
diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py
--- a/pypy/module/thread/os_lock.py
+++ b/pypy/module/thread/os_lock.py
@@ -121,10 +121,10 @@
     def descr__repr__(self, space):
         classname = space.getfulltypename(self)
         if self._is_locked():
-            locked = u"locked"
+            locked = "locked"
         else:
-            locked = u"unlocked"
-        return self.getrepr(space, u'%s %s object' % (locked, classname))
+            locked = "unlocked"
+        return self.getrepr(space, '%s %s object' % (locked, classname))
 
 Lock.typedef = TypeDef(
     "_thread.lock",
@@ -189,10 +189,10 @@
     def descr__repr__(self, space):
         classname = space.getfulltypename(self)
         if self.rlock_count == 0:
-            locked = u"unlocked"
+            locked = "unlocked"
         else:
-            locked = u"locked"
-        return self.getrepr(space, u'%s %s object owner=%d count=%d' % (
+            locked = "locked"
+        return self.getrepr(space, '%s %s object owner=%d count=%d' % (
             locked, classname, self.rlock_owner, self.rlock_count))
 
     @unwrap_spec(blocking=int, timeout=float)
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -267,9 +267,9 @@
 
     def descr_repr(self, space):
         if self.view is None:
-            return self.getrepr(space, u'released memory')
+            return self.getrepr(space, 'released memory')
         else:
-            return self.getrepr(space, u'memory')
+            return self.getrepr(space, 'memory')
 
     def descr_hash(self, space):
         if self._hash == -1:

From pypy.commits at gmail.com  Mon Aug  6 03:19:17 2018
From: pypy.commits at gmail.com (mattip)
Date: Mon, 06 Aug 2018 00:19:17 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: decode needs an argument
Message-ID: <5b67f675.1c69fb81.cecba.964d@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94957:a7da1446b505
Date: 2018-08-05 15:30 -0700
http://bitbucket.org/pypy/pypy/changeset/a7da1446b505/

Log:	decode needs an argument

diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py
--- a/pypy/module/_winreg/interp_winreg.py
+++ b/pypy/module/_winreg/interp_winreg.py
@@ -222,7 +222,7 @@
     if typ != rwinreg.REG_SZ:
         raise oefmt(space.w_ValueError, "Type must be winreg.REG_SZ")
     hkey = hkey_w(w_hkey, space)
-    with rffi.scoped_unicode2wcharp(space.utf8_w(w_subkey).decode()) as subkey:
+    with rffi.scoped_unicode2wcharp(space.utf8_w(w_subkey).decode('utf8')) as subkey:
         c_subkey = rffi.cast(rffi.CCHARP, subkey)
         with rffi.scoped_unicode2wcharp(value) as dataptr:
             c_dataptr = rffi.cast(rffi.CCHARP, dataptr)
@@ -246,7 +246,7 @@
     if space.is_w(w_subkey, space.w_None):
         subkey = None
     else:
-        subkey = space.utf8_w(w_subkey).decode()
+        subkey = space.utf8_w(w_subkey).decode('utf8')
     with rffi.scoped_unicode2wcharp(subkey) as wide_subkey:
         c_subkey = rffi.cast(rffi.CCHARP, wide_subkey)
         with lltype.scoped_alloc(rwin32.PLONG.TO, 1) as bufsize_p:
@@ -296,7 +296,7 @@
             buf = lltype.malloc(rffi.CCHARP.TO, buflen, flavor='raw')
             buf[0] = '\0'
         else:
-            buf = rffi.unicode2wcharp(space.utf8_w(w_value).decode())
+            buf = rffi.unicode2wcharp(space.utf8_w(w_value).decode('utf8'))
             buf = rffi.cast(rffi.CCHARP, buf)
             buflen = (space.len_w(w_value) * 2) + 1
 
@@ -314,7 +314,7 @@
             while True:
                 try:
                     w_item = space.next(w_iter)
-                    item = space.utf8_w(w_item).decode()
+                    item = space.utf8_w(w_item).decode('utf8')
                     strings.append(item)
                     buflen += 2 * (len(item) + 1)
                 except OperationError as e:
@@ -455,7 +455,7 @@
     if space.is_w(w_subkey, space.w_None):
         subkey = None
     else:
-        subkey = space.utf8_w(w_subkey).decode()
+        subkey = space.utf8_w(w_subkey).decode('utf8')
     null_dword = lltype.nullptr(rwin32.LPDWORD.TO)
     with rffi.scoped_unicode2wcharp(subkey) as wide_subkey:
         c_subkey = rffi.cast(rffi.CCHARP, wide_subkey)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1716,7 +1716,7 @@
     raise_import_error(space, space.newtext(msg), w_name, w_path)
 
 def get_init_name(space, w_name):
-    name_u = space.utf8_w(w_name).decode()
+    name_u = space.utf8_w(w_name).decode('utf8')
     basename_u = name_u.split(u'.')[-1]
     try:
         basename = basename_u.encode('ascii')
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -254,8 +254,8 @@
 
     def descr_method_repr(self):
         return self.getrepr(
-            self.space, u"built-in method '%s' of '%s' object" %
-            (self.name.decode('utf-8'), self.w_objclass.getname(self.space)))
+            self.space, "built-in method '%s' of '%s' object" %
+            (self.name, self.w_objclass.getname(self.space)))
 
 
 class W_PyCWrapperObject(W_Root):
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -141,7 +141,7 @@
             argv = space.sys.get('argv')
             if space.len_w(argv):
                 argv0 = space.getitem(argv, space.newint(0))
-                progname = space.utf8_w(argv0).decode()
+                progname = space.utf8_w(argv0).decode('utf8')
             else:
                 progname = u"pypy3"
             self.programname = rffi.unicode2wcharp(progname)
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -383,9 +383,12 @@
                     # will never be called.  This causes RPython
                     # problems.  Avoid it with the nonconst hack.
                     if not allow_surrogates or nonconst.NonConstant(False):
+                        utf8 = s
+                        if isinstance(s, unicode):
+                            utf8 = s.encode('utf8')
                         ru, rs, pos = errorhandler(errors, 'utf8',
                                                    'surrogates not allowed',
-                                                   s, pos-1, pos)
+                                                   utf8, pos-1, pos)
                         if rs is not None:
                             # py3k only
                             result.append(rs)
@@ -396,7 +399,7 @@
                             else:
                                 errorhandler('strict', 'utf8',
                                              'surrogates not allowed',
-                                             s, pos-1, pos)
+                                             utf8, pos-1, pos)
                         continue
                     # else: Fall through and handles isolated high surrogates
                 result.append((chr((0xe0 | (ch >> 12)))))

From pypy.commits at gmail.com  Mon Aug  6 03:19:20 2018
From: pypy.commits at gmail.com (mattip)
Date: Mon, 06 Aug 2018 00:19:20 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: more internal unicode->utf8
Message-ID: <5b67f678.1c69fb81.8bb11.e539@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94958:ff00333ba97b
Date: 2018-08-05 15:31 -0700
http://bitbucket.org/pypy/pypy/changeset/ff00333ba97b/

Log:	more internal unicode->utf8

diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -554,7 +554,7 @@
             except OperationError as e:
                 if not e.match(space, space.w_TypeError):
                     raise
-                name = u'?'
+                name = '?'
         objrepr = space.utf8_w(space.repr(self.w_instance))
         s = b'<bound method %s of %s>' % (name, objrepr)
         return space.newtext(s)
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -397,7 +397,7 @@
            self.frame is not None and \
            self.frame.last_instr == -1:
             space = self.space
-            msg = u"coroutine '%s' was never awaited" % self.get_qualname()
+            msg = "coroutine '%s' was never awaited" % self.get_qualname()
             space.warn(space.newtext(msg), space.w_RuntimeWarning)
         GeneratorOrCoroutine._finalize_(self)
 

From pypy.commits at gmail.com  Mon Aug  6 03:19:22 2018
From: pypy.commits at gmail.com (mattip)
Date: Mon, 06 Aug 2018 00:19:22 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: avoid converting to unicode in
 _isidentifier
Message-ID: <5b67f67a.1c69fb81.bdd4c.8d55@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94959:c1b6ab618fdd
Date: 2018-08-05 15:51 -0700
http://bitbucket.org/pypy/pypy/changeset/c1b6ab618fdd/

Log:	avoid converting to unicode in _isidentifier

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -624,7 +624,7 @@
         return space.newbool(cased)
 
     def descr_isidentifier(self, space):
-        return space.newbool(_isidentifier(self._utf8.decode('utf8')))
+        return space.newbool(_isidentifier(self._utf8))
 
     def descr_startswith(self, space, w_prefix, w_start=None, w_end=None):
         start, end = self._unwrap_and_compute_idx_params(space, w_start, w_end)
@@ -1162,11 +1162,13 @@
     # to check just for these, except that _ must be allowed as starting
     # an identifier.
     first = u[0]
-    if not (unicodedb.isxidstart(ord(first)) or first == u'_'):
+    it = rutf8.Utf8StringIterator(u)
+    code = it.next()
+    if not (unicodedb.isxidstart(code) or first == u'_'):
         return False
 
-    for i in range(1, len(u)):
-        if not unicodedb.isxidcontinue(ord(u[i])):
+    for ch in it:
+        if not unicodedb.isxidcontinue(ch):
             return False
     return True
 

From pypy.commits at gmail.com  Mon Aug  6 03:19:25 2018
From: pypy.commits at gmail.com (mattip)
Date: Mon, 06 Aug 2018 00:19:25 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: unicode -> utf8,
 non-negative len
Message-ID: <5b67f67d.1c69fb81.d68c.11c5@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94960:31049cf23000
Date: 2018-08-05 19:20 -0700
http://bitbucket.org/pypy/pypy/changeset/31049cf23000/

Log:	unicode -> utf8, non-negative len

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -38,7 +38,7 @@
         # 'qualname' is a unicode string
         if self._qualname is not None:
             return self._qualname
-        return self.get_name().decode('utf-8')
+        return self.get_name()
 
     def descr__repr__(self, space):
         addrstring = self.getaddrstring(space)
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -452,8 +452,8 @@
     def repr(self, space):
         space = self.space
         # co_name should be an identifier
-        name = self.co_name.decode('utf-8')
+        name = self.co_name
         fn = space.utf8_w(self.w_filename)
         return space.newtext(b'<code object %s at 0x%s, file "%s", line %d>' % (
-            name, unicode(self.getaddrstring(space)), fn,
+            name, self.getaddrstring(space), fn,
             -1 if self.co_firstlineno == 0 else self.co_firstlineno))
diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -79,7 +79,7 @@
     except UnicodeDecodeError:
         return -1
     from pypy.objspace.std.unicodeobject import _isidentifier
-    return _isidentifier(u)
+    return _isidentifier(token)
 
 
 DUMMY_DFA = automata.DFA([], [])
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -222,6 +222,7 @@
                 raise oefmt(self.space.w_ValueError,
                             "%s out of range for conversion to unicode: %s",
                             self.name, hex(e.ordinal))
+        assert lgt >= 0
         return self.space.newutf8(utf8, lgt)
 
 
diff --git a/pypy/module/_multibytecodec/interp_multibytecodec.py b/pypy/module/_multibytecodec/interp_multibytecodec.py
--- a/pypy/module/_multibytecodec/interp_multibytecodec.py
+++ b/pypy/module/_multibytecodec/interp_multibytecodec.py
@@ -78,6 +78,7 @@
             space.newtext(e.reason)]))
 
 def wrap_unicodeencodeerror(space, e, input, inputlen, name):
+    assert inputlen >= 0
     raise OperationError(
         space.w_UnicodeEncodeError,
         space.newtuple([
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1711,7 +1711,7 @@
             look_for = also_look_for
     assert look_for is not None
     msg = b"function %s not found in library %s" % (
-        look_for.decode('utf-8'), space.utf8_w(space.newfilename(path)))
+        look_for, space.utf8_w(space.newfilename(path)))
     w_path = space.newfilename(path)
     raise_import_error(space, space.newtext(msg), w_name, w_path)
 

From pypy.commits at gmail.com  Mon Aug  6 03:19:27 2018
From: pypy.commits at gmail.com (mattip)
Date: Mon, 06 Aug 2018 00:19:27 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: fix unicodehelper errorhandlers
Message-ID: <5b67f67f.1c69fb81.49ae7.b723@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94961:18d61ca77776
Date: 2018-08-05 23:27 -0700
http://bitbucket.org/pypy/pypy/changeset/18d61ca77776/

Log:	fix unicodehelper errorhandlers

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -32,7 +32,7 @@
 def decode_never_raise(errors, encoding, msg, s, startingpos, endingpos):
     assert startingpos >= 0
     ux = ['\ux' + hex(ord(x))[2:].upper() for x in s[startingpos:endingpos]]
-    return ''.join(ux), endingpos, endingpos
+    return ''.join(ux), endingpos
 
 @specialize.memo()
 def encode_error_handler(space):
@@ -224,6 +224,10 @@
     # cannot be ASCII, cannot have surrogates, I believe
     return res.build(), len(s), len(s)
 
+def utf8_encode_utf_8(s, errors, errorhandler):
+    # needed by tests
+    return s
+
 def utf8_encode_latin_1(s, errors, errorhandler):
     try:
         rutf8.check_ascii(s)
@@ -295,7 +299,7 @@
     return result.build()
 
 if sys.platform == 'win32':
-    def utf8_encode_mbcs(s, slen, errors, errorhandler):
+    def utf8_encode_mbcs(s, errors, errorhandler):
         s = s.decode('utf-8')
         res = unicode_encode_mbcs(s, slen, errors, errorhandler)
         return res
@@ -606,7 +610,7 @@
                                   errorhandler=None):
     size = len(s)
     if size == 0:
-        return '', 0
+        return '', 0, 0
 
     builder = rutf8.Utf8StringBuilder(size)
     pos = 0
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -946,13 +946,12 @@
 
     unicode_name_handler = state.get_unicodedata_handler(space)
 
-    result, lgt = unicodehelper.str_decode_unicode_escape(
+    result, lgt, u_len = unicodehelper.str_decode_unicode_escape(
         string, errors,
         final, state.decode_error_handler,
         unicode_name_handler)
 
-    s_len = len(string)
-    return space.newtuple([space.newutf8(result, lgt), space.newint(s_len)])
+    return space.newtuple([space.newutf8(result, lgt), space.newint(u_len)])
 
 # ____________________________________________________________
 # Raw Unicode escape (accepts bytes or str)
@@ -964,9 +963,8 @@
         errors = 'strict'
     final = space.is_true(w_final)
     state = space.fromcache(CodecState)
-    result, lgt = runicode.str_decode_raw_unicode_escape(
-        string, len(string), errors,
-        final, state.decode_error_handler)
+    result, lgt, u_len = unicodehelper.str_decode_raw_unicode_escape(
+        string, errors, final, state.decode_error_handler)
     return space.newtuple([space.newtext(result), space.newint(lgt)])
 
 # ____________________________________________________________
diff --git a/pypy/module/_codecs/test/test_locale.py b/pypy/module/_codecs/test/test_locale.py
--- a/pypy/module/_codecs/test/test_locale.py
+++ b/pypy/module/_codecs/test/test_locale.py
@@ -41,7 +41,7 @@
         utf8_encoder = self.getencoder('utf-8')
         for val in u'foo', u' &#26085;&#26412;', u'\U0001320C':
             assert (locale_encoder(val).encode('utf8') ==
-                    utf8_encoder(val, 'strict', True, None))
+                    utf8_encoder(val, 'strict', None))
 
     def test_encode_locale_errorhandler(self):
         self.setlocale("en_US.UTF-8")
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -30,7 +30,7 @@
 
 class W_UnicodeObject(W_Root):
     import_from_mixin(StringMethods)
-    _immutable_fields_ = ['_utf8']
+    _immutable_fields_ = ['_utf8', '_length']
 
     @enforceargs(utf8str=str)
     def __init__(self, utf8str, length):

From pypy.commits at gmail.com  Mon Aug  6 03:19:29 2018
From: pypy.commits at gmail.com (mattip)
Date: Mon, 06 Aug 2018 00:19:29 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: more translation fixes
Message-ID: <5b67f681.1c69fb81.b7ea5.38ca@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94962:a39a0158002f
Date: 2018-08-06 00:15 -0700
http://bitbucket.org/pypy/pypy/changeset/a39a0158002f/

Log:	more translation fixes

diff --git a/pypy/interpreter/astcompiler/fstring.py b/pypy/interpreter/astcompiler/fstring.py
--- a/pypy/interpreter/astcompiler/fstring.py
+++ b/pypy/interpreter/astcompiler/fstring.py
@@ -264,9 +264,8 @@
         space = astbuilder.space
         literal = parsestring.decode_unicode_utf8(space, literal, 0,
                                                   len(literal))
-        return unicodehelper.decode_unicode_escape(space, literal)
-    else:
-        return literal.decode('utf-8')
+        literal, lgt, _ = unicodehelper.decode_unicode_escape(space, literal)
+    return literal.decode('utf-8')
 
 
 def fstring_find_literal_and_expr(astbuilder, fstr, atom_node, rec):
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -223,6 +223,8 @@
     w_start = space.getattr(w_exc, space.newtext('start'))
     w_end = space.getattr(w_exc, space.newtext('end'))
     size = space.int_w(w_end) - space.int_w(w_start)
+    if size < 0:
+        size = 0
     if space.isinstance_w(w_exc, space.w_UnicodeEncodeError):
         text = '?' * size
         return space.newtuple([space.newutf8(text, size), w_end])
@@ -315,7 +317,7 @@
             try:
                 name = unicodedb.name(oc)
             except KeyError:
-                raw_unicode_escape_helper_unicode(builder, oc)
+                unicodehelper.raw_unicode_escape_helper(builder, oc)
             else:
                 builder.append('\\N{')
                 builder.append(name)
diff --git a/pypy/module/_io/interp_stringio.py b/pypy/module/_io/interp_stringio.py
--- a/pypy/module/_io/interp_stringio.py
+++ b/pypy/module/_io/interp_stringio.py
@@ -156,10 +156,11 @@
     def descr_getstate(self, space):
         w_initialval = self.getvalue_w(space)
         w_dict = space.call_method(self.w_dict, "copy")
-        if self.readnl is None:
+        readnl = self.readnl
+        if readnl is None:
             w_readnl = space.w_None
         else:
-            w_readnl = space.str(space.newutf8(self.readnl, get_utf8_length(self.readnl)))  # YYY
+            w_readnl = space.str(space.newutf8(readnl, get_utf8_length(readnl)))  # YYY
         return space.newtuple([
             w_initialval, w_readnl, space.newint(self.buf.pos), w_dict
         ])

From pypy.commits at gmail.com  Tue Aug  7 16:03:21 2018
From: pypy.commits at gmail.com (mattip)
Date: Tue, 07 Aug 2018 13:03:21 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: seperate runicode and pypy
 error handlers, simplifies unicode/utf8 return vals
Message-ID: <5b69fb09.1c69fb81.94d90.c998@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94963:06beac29ff13
Date: 2018-08-06 21:57 -0700
http://bitbucket.org/pypy/pypy/changeset/06beac29ff13/

Log:	seperate runicode and pypy error handlers, simplifies unicode/utf8
	return vals

diff --git a/pypy/interpreter/pyparser/error.py b/pypy/interpreter/pyparser/error.py
--- a/pypy/interpreter/pyparser/error.py
+++ b/pypy/interpreter/pyparser/error.py
@@ -35,13 +35,17 @@
             # XXX do the right thing about continuation lines, which
             # XXX are their own fun, sometimes giving offset >
             # XXX len(self.text) for example (right now, avoid crashing)
+            def replace_error_handler(errors, encoding, msg, s, startpos, endpos):
+                # must return unicode
+                return u'\ufffd', endpos
             if offset > len(self.text):
                 offset = len(self.text)
-            text, _ = str_decode_utf_8(self.text, offset, 'replace')
+            text, _ = str_decode_utf_8(self.text, offset,
+                             'replace', errorhandler=replace_error_handler)
             offset = len(text)
             if len(self.text) != offset:
                 text, _ = str_decode_utf_8(self.text, len(self.text),
-                                           'replace')
+                             'replace', errorhandler=replace_error_handler)
             w_text = space.newtext(text)
         return space.newtuple([
             space.newtext(self.msg),
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -27,6 +27,8 @@
                                              space.newint(startingpos),
                                              space.newint(endingpos),
                                              space.newtext(msg)]))
+        # make annotator happy
+        return '', 0
     return raise_unicode_exception_decode
 
 def decode_never_raise(errors, encoding, msg, s, startingpos, endingpos):
@@ -86,8 +88,7 @@
         from pypy.module._codecs.locale import (
             str_decode_locale_surrogateescape)
         bytes = space.bytes_w(w_string)
-        uni = str_decode_locale_surrogateescape(
-            bytes, errorhandler=decode_error_handler(space))
+        uni = str_decode_locale_surrogateescape(bytes)
     else:
         from pypy.module.sys.interp_encoding import getfilesystemencoding
         return space.call_method(w_string, 'decode',
@@ -301,11 +302,15 @@
 if sys.platform == 'win32':
     def utf8_encode_mbcs(s, errors, errorhandler):
         s = s.decode('utf-8')
+        if errorhandler is None:
+            errorhandler = encode_error_handler(space)
         res = unicode_encode_mbcs(s, slen, errors, errorhandler)
         return res
         
     def str_decode_mbcs(s, errors, final, errorhandler):
         slen = len(s)
+        if errorhandler is None:
+            errorhandler = decode_error_handler(space) 
         res, size = str_decode_mbcs(s, slen, final=final, errors=errors,
                                            errorhandler=errorhandler)
         return res.encode('utf8'), len(res)
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -3,7 +3,7 @@
 from rpython.rlib.objectmodel import we_are_translated, not_rpython
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib import runicode
-from rpython.rlib.runicode import raw_unicode_escape_helper_unicode
+from rpython.rlib.runicode import raw_unicode_escape_helper
 from rpython.rlib import rutf8
 
 from pypy.interpreter.error import OperationError, oefmt
@@ -39,8 +39,8 @@
             the unicode characters, not into the position of utf8 bytes,
             so it needs to be converted by the codec
 
-            Returns (unicode_or_none, str_or_none, newpos) as error
-            handlers may return unicode or on Python 3, bytes.
+            Returns (str_or_none, newpos) as error
+            handlers used outside runicode return utf8
             """
             w_errorhandler = lookup_error(space, errors)
             if decode:
@@ -275,11 +275,11 @@
         start = space.int_w(space.getattr(w_exc, space.newtext('start')))
         w_end = space.getattr(w_exc, space.newtext('end'))
         end = space.int_w(w_end)
-        builder = UnicodeBuilder()
+        builder = StringBuilder()
         pos = start
         while pos < end:
             oc = ord(obj[pos])
-            raw_unicode_escape_helper_unicode(builder, oc)
+            raw_unicode_escape_helper(builder, oc)
             pos += 1
         return space.newtuple([space.newtext(builder.build()), w_end])
     elif space.isinstance_w(w_exc, space.w_UnicodeDecodeError):
@@ -287,11 +287,11 @@
         start = space.int_w(space.getattr(w_exc, space.newtext('start')))
         w_end = space.getattr(w_exc, space.newtext('end'))
         end = space.int_w(w_end)
-        builder = UnicodeBuilder()
+        builder = StringBuilder()
         pos = start
         while pos < end:
             oc = ord(obj[pos])
-            raw_unicode_escape_helper_unicode(builder, oc)
+            raw_unicode_escape_helper(builder, oc)
             pos += 1
         return space.newtuple([space.newtext(builder.build()), w_end])
     else:
diff --git a/pypy/module/_cppyy/test/test_zjit.py b/pypy/module/_cppyy/test/test_zjit.py
--- a/pypy/module/_cppyy/test/test_zjit.py
+++ b/pypy/module/_cppyy/test/test_zjit.py
@@ -71,9 +71,7 @@
         self.name = name
         self.__name__ = name
     def getname(self, space, name):
-        if sys.hexversion < 0x3000000:
-            return self.name
-        return unicode(self.name)
+        return self.name
 class FakeBuffer(FakeBase):
     typedname = "buffer"
     def __init__(self, val):

From pypy.commits at gmail.com  Tue Aug  7 16:03:23 2018
From: pypy.commits at gmail.com (mattip)
Date: Tue, 07 Aug 2018 13:03:23 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: remove unicode conversions
Message-ID: <5b69fb0b.1c69fb81.3e00c.8ae7@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94964:7b350c74d353
Date: 2018-08-06 21:59 -0700
http://bitbucket.org/pypy/pypy/changeset/7b350c74d353/

Log:	remove unicode conversions

diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -134,7 +134,7 @@
     def type(self, obj):
         class Type:
             def getname(self, space):
-                return unicode(type(obj).__name__)
+                return type(obj).__name__
         return Type()
 
 
diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -597,7 +597,7 @@
 
     def getname(self, space): 
         # for the benefit of Method/instancemethod
-        return capi.c_method_name(space, self.functions[0].cppmethod).decode('latin-1')
+        return capi.c_method_name(space, self.functions[0].cppmethod)
 
     def __repr__(self):
         return "W_CPPOverload(%s)" % [f.prototype() for f in self.functions]
@@ -832,7 +832,7 @@
         return self.getitem_impl(self.name, args_w)
 
     def getname(self, space):
-        return self.name.decode('latin-1')
+        return self.name
 
     def __repr__(self):
         return "W_CPPTemplateOverload(%s)" % [f.prototype() for f in self.functions]
@@ -890,7 +890,7 @@
         return self.getitem_impl(self.name, args_w)
 
     def getname(self, space):
-        return self.name.decode('latin-1')
+        return self.name
 
     def __repr__(self):
         return "W_CPPTemplateStaticOverload(%s)" % [f.prototype() for f in self.functions]
diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -849,7 +849,7 @@
                 haslf = True
         if haslf and self.writetranslate and self.writenl:
             w_text = space.call_method(w_text, "replace", space.newutf8('\n', 1),
-                                       space.newutf8(self.writenl, get_utf8_length(self.writenl)))
+                               space.newutf8(self.writenl, get_utf8_length(self.writenl)))
             text = space.utf8_w(w_text)
 
         needflush = False
diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py
--- a/pypy/module/_lsprof/interp_lsprof.py
+++ b/pypy/module/_lsprof/interp_lsprof.py
@@ -207,13 +207,12 @@
         if isinstance(w_type, W_TypeObject):
             w_realclass, _ = space.lookup_in_type_where(w_type, name)
             if isinstance(w_realclass, W_TypeObject):
-                class_name = w_realclass.name.decode('utf-8')
+                class_name = w_realclass.name
     else:
         name = '?'
     if class_name is None:
         class_name = w_type.getname(space)    # if the rest doesn't work
-    return u"<method '%s' of '%s' objects>" % (name.decode('utf-8'),
-                                               class_name)
+    return b"<method '%s' of '%s' objects>" % (name, class_name)
 
 
 def create_spec_for_function(space, w_func):
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -105,7 +105,7 @@
 
     def repr_w(self):
         space = self.space
-        u = space.realunicode_w(space.repr(self.w_pattern))
+        u = space.utf8_w(space.repr(self.w_pattern))
         if len(u) > 200:
             u = u[:200]
         flag_items = []
diff --git a/pypy/module/_warnings/interp_warnings.py b/pypy/module/_warnings/interp_warnings.py
--- a/pypy/module/_warnings/interp_warnings.py
+++ b/pypy/module/_warnings/interp_warnings.py
@@ -271,11 +271,11 @@
     if not line:
         return
 
-    message = u"\n"
+    message = "\n"
     for i in range(len(line)):
         c = line[i]
-        if c not in u' \t\014':
-            message = u"  %s\n" % (line[i:],)
+        if c not in ' \t\014':
+            message = "  %s\n" % (line[i:],)
             break
     space.call_method(w_stderr, "write", space.newtext(message))
 
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -82,7 +82,7 @@
         return w_some_obj()
 
     def getname(self, space):
-        return self.name.decode('utf-8')
+        return self.name
 
 def w_some_obj():
     if NonConstant(False):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -401,7 +401,7 @@
         return self.newtext(s)
 
     # XXX find where length is annotated as negative int
-    @signature(types.any(), types.str(), types.int_nonneg(), returns=types.any())
+    #@signature(types.any(), types.str(), types.int_nonneg(), returns=types.any())
     def newutf8(self, utf8s, length):
         assert isinstance(utf8s, str)
         return W_UnicodeObject(utf8s, length)

From pypy.commits at gmail.com  Tue Aug  7 16:03:26 2018
From: pypy.commits at gmail.com (mattip)
Date: Tue, 07 Aug 2018 13:03:26 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: unicode0_w -> utf8_0_w
Message-ID: <5b69fb0e.1c69fb81.b0afc.bd74@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94965:beaa8a7c7e84
Date: 2018-08-06 21:59 -0700
http://bitbucket.org/pypy/pypy/changeset/beaa8a7c7e84/

Log:	unicode0_w -> utf8_0_w

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1717,6 +1717,16 @@
     def utf8_w(self, w_obj):
         return w_obj.utf8_w(self)
 
+    def utf8_0_w(self, w_obj):
+        "Like utf_w, but rejects strings with NUL bytes."
+        from rpython.rlib import rstring
+        result = w_obj.utf8_w(self)
+        if '\x00' in result:
+            raise oefmt(self.w_TypeError,
+                        "argument must be a string without NUL "
+                        "characters")
+        return rstring.assert_str0(result)
+
     def convert_to_w_unicode(self, w_obj):
         return w_obj.convert_to_w_unicode(self)
 
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -595,7 +595,7 @@
         w_output = space.fsdecode(w_obj)
         if not space.isinstance_w(w_output, space.w_unicode):
             raise oefmt(space.w_TypeError, "decoder failed to return unicode")
-    data = space.unicode0_w(w_output)  # Check for NUL bytes
+    data = space.utf8_0_w(w_output)  # Check for NUL bytes
     result[0] = make_ref(space, w_output)
     return Py_CLEANUP_SUPPORTED
 
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -141,7 +141,7 @@
             "can't specify None for path argument")
     if _WIN32:
         try:
-            path_u = space.unicode0_w(w_value)
+            path_u = space.utf8_0_w(w_value)
             return Path(-1, None, path_u, w_value)
         except OperationError:
             pass
diff --git a/pypy/module/posix/interp_scandir.py b/pypy/module/posix/interp_scandir.py
--- a/pypy/module/posix/interp_scandir.py
+++ b/pypy/module/posix/interp_scandir.py
@@ -269,7 +269,7 @@
 
         def get_stat_or_lstat(self, follow_symlinks):     # 'follow_symlinks' ignored
             if not self.stat_cached:
-                path = self.space.unicode0_w(self.fget_path(self.space))
+                path = self.space.utf8_0_w(self.fget_path(self.space))
                 self.d_stat = rposix_stat.stat(path)
                 self.stat_cached = True
             return self.d_stat

From pypy.commits at gmail.com  Tue Aug  7 16:03:27 2018
From: pypy.commits at gmail.com (mattip)
Date: Tue, 07 Aug 2018 13:03:27 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: unicode -> utf8
Message-ID: <5b69fb0f.1c69fb81.7a093.172b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94966:9099e13cc646
Date: 2018-08-07 00:17 -0700
http://bitbucket.org/pypy/pypy/changeset/9099e13cc646/

Log:	unicode -> utf8

diff --git a/pypy/module/_codecs/locale.py b/pypy/module/_codecs/locale.py
--- a/pypy/module/_codecs/locale.py
+++ b/pypy/module/_codecs/locale.py
@@ -57,7 +57,7 @@
                 if errorpos == -1:
                     raise MemoryError
                 errmsg = _errmsg("pypy_wchar2char")
-                errorhandler('strict', 'filesystemencoding', errmsg, u.encode('utf8'),
+                errorhandler('strict', 'filesystemencoding', errmsg, u,
                              errorpos, errorpos + 1)
             return rffi.charp2str(sbuf)
         finally:
diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -856,7 +856,7 @@
         text_needflush = False
         if self.write_through:
             text_needflush = True
-        if self.line_buffering and (haslf or text.find(u'\r') >= 0):
+        if self.line_buffering and (haslf or text.find('\r') >= 0):
             needflush = True
 
         # XXX What if we were just reading?
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -1454,7 +1454,7 @@
     names = [cls.getname(space) for cls in cycle]
     # Can't use oefmt() here, since names is a list of unicodes
     raise OperationError(space.w_TypeError, space.newtext(
-        u"cycle among base classes: " + u' < '.join(names)))
+        "cycle among base classes: " + ' < '.join(names)))
 
 
 class TypeCache(SpaceCache):

From pypy.commits at gmail.com  Tue Aug  7 16:03:30 2018
From: pypy.commits at gmail.com (mattip)
Date: Tue, 07 Aug 2018 13:03:30 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: revert wrong rpython change
Message-ID: <5b69fb12.1c69fb81.2a069.a380@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94967:739c19660589
Date: 2018-08-07 09:14 -0700
http://bitbucket.org/pypy/pypy/changeset/739c19660589/

Log:	revert wrong rpython change

diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -383,12 +383,9 @@
                     # will never be called.  This causes RPython
                     # problems.  Avoid it with the nonconst hack.
                     if not allow_surrogates or nonconst.NonConstant(False):
-                        utf8 = s
-                        if isinstance(s, unicode):
-                            utf8 = s.encode('utf8')
                         ru, rs, pos = errorhandler(errors, 'utf8',
                                                    'surrogates not allowed',
-                                                   utf8, pos-1, pos)
+                                                   s, pos-1, pos)
                         if rs is not None:
                             # py3k only
                             result.append(rs)
@@ -399,7 +396,7 @@
                             else:
                                 errorhandler('strict', 'utf8',
                                              'surrogates not allowed',
-                                             utf8, pos-1, pos)
+                                             s, pos-1, pos)
                         continue
                     # else: Fall through and handles isolated high surrogates
                 result.append((chr((0xe0 | (ch >> 12)))))

From pypy.commits at gmail.com  Tue Aug  7 16:03:32 2018
From: pypy.commits at gmail.com (mattip)
Date: Tue, 07 Aug 2018 13:03:32 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: avoid untranslatable
 unicodehelper.encode_utf8, add TODO note
Message-ID: <5b69fb14.1c69fb81.ef517.98b6@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94968:9fa79905a9c9
Date: 2018-08-07 09:16 -0700
http://bitbucket.org/pypy/pypy/changeset/9fa79905a9c9/

Log:	avoid untranslatable unicodehelper.encode_utf8, add TODO note

diff --git a/TODO b/TODO
--- a/TODO
+++ b/TODO
@@ -4,3 +4,4 @@
 * improve performance of splitlines
 * fix _pypyjson to not use a wrapped dict when decoding an object
 * make sure we review all the places that call ord(unichr) to check for ValueErrors
+* rewrite unicodeobject.unicode_to_decimal_w to only use utf8 encoded bytes
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -1882,10 +1882,16 @@
     if not isinstance(w_unistr, W_UnicodeObject):
         raise oefmt(space.w_TypeError, "expected unicode, got '%T'", w_unistr)
     value = _rpy_unicode_to_decimal_w(space, w_unistr.utf8_w(space).decode('utf8'))
-    return unicodehelper.encode_utf8(space, value,
-                                     allow_surrogates=allow_surrogates)
+    # XXX this is the only place in the code that this funcion is called.
+    # It does not translate, since it uses a pypy-level error handler
+    # to throw the UnicodeEncodeError not the rpython default handler
+    #return unicodehelper.encode_utf8(space, value,
+    #                                 allow_surrogates=allow_surrogates)
+    assert isinstance(value, unicode)
+    return value.encode('utf8')
 
 def _rpy_unicode_to_decimal_w(space, unistr):
+    # XXX rewrite this to accept a utf8 string and use a StringBuilder
     result = [u'\0'] * len(unistr)
     for i in xrange(len(unistr)):
         uchr = ord(unistr[i])

From pypy.commits at gmail.com  Tue Aug  7 16:03:34 2018
From: pypy.commits at gmail.com (mattip)
Date: Tue, 07 Aug 2018 13:03:34 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: decode takes unicode input
Message-ID: <5b69fb16.1c69fb81.2a069.a381@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94969:c27f5e8d2da0
Date: 2018-08-07 09:16 -0700
http://bitbucket.org/pypy/pypy/changeset/c27f5e8d2da0/

Log:	decode takes unicode input

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -45,12 +45,14 @@
             w_errorhandler = lookup_error(space, errors)
             if decode:
                 w_cls = space.w_UnicodeDecodeError
+                assert isinstance(input, str)
                 w_input = space.newbytes(input)
                 length = len(input)
             else:
                 w_cls = space.w_UnicodeEncodeError
-                length = rutf8.codepoints_in_utf8(input)
-                w_input = space.newtext((input, length, length))
+                length = len(input)
+                assert isinstance(input, unicode)
+                w_input = space.newtext((input.encode('utf8'), length, length))
             w_exc =  space.call_function(
                 w_cls,
                 space.newtext(encoding),
@@ -721,12 +723,11 @@
     if errors is None:
         errors = 'strict'
     state = space.fromcache(CodecState)
-    # NB. can't call unicode_encode_utf_8() directly because that's
-    # an @elidable function nowadays.  Instead, we need the _impl().
-    # (The problem is the errorhandler, which calls arbitrary Python.)
-    result = runicode.unicode_encode_utf_8_impl(
-        utf8, lgt, errors, state.encode_error_handler,
-        allow_surrogates=False)
+    #result = runicode.unicode_encode_utf_8_impl(
+    #    utf8, lgt, errors, state.encode_error_handler,
+    #    allow_surrogates=False)
+    result = unicodehelper.utf8_encode_utf_8(utf8, errors,
+                     state.encode_error_handler, allow_surrogates=False)
     return space.newtuple([space.newbytes(result), space.newint(lgt)])
 
 @unwrap_spec(string='bufferstr', errors='text_or_none',

From pypy.commits at gmail.com  Tue Aug  7 16:03:36 2018
From: pypy.commits at gmail.com (mattip)
Date: Tue, 07 Aug 2018 13:03:36 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: avoid elidible fail,
 should revisit and figure out why
Message-ID: <5b69fb18.1c69fb81.9b8ec.f6b8@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94970:fbb06dc84f8e
Date: 2018-08-07 09:20 -0700
http://bitbucket.org/pypy/pypy/changeset/fbb06dc84f8e/

Log:	avoid elidible fail, should revisit and figure out why

diff --git a/TODO b/TODO
--- a/TODO
+++ b/TODO
@@ -5,3 +5,4 @@
 * fix _pypyjson to not use a wrapped dict when decoding an object
 * make sure we review all the places that call ord(unichr) to check for ValueErrors
 * rewrite unicodeobject.unicode_to_decimal_w to only use utf8 encoded bytes
+* revisit why runicode import str_decode_utf_8_impl needed instead of runicode import str_decode_utf_8
diff --git a/pypy/interpreter/pyparser/error.py b/pypy/interpreter/pyparser/error.py
--- a/pypy/interpreter/pyparser/error.py
+++ b/pypy/interpreter/pyparser/error.py
@@ -29,7 +29,7 @@
                     except:  # we can't allow any exceptions here!
                         return None""")
         elif self.text is not None:
-            from rpython.rlib.runicode import str_decode_utf_8
+            from rpython.rlib.runicode import str_decode_utf_8_impl
             # self.text may not be UTF-8 in case of decoding errors.
             # adjust the encoded text offset to a decoded offset
             # XXX do the right thing about continuation lines, which
@@ -40,12 +40,12 @@
                 return u'\ufffd', endpos
             if offset > len(self.text):
                 offset = len(self.text)
-            text, _ = str_decode_utf_8(self.text, offset,
-                             'replace', errorhandler=replace_error_handler)
+            text, _ = str_decode_utf_8_impl(self.text, offset,
+                             'replace', False, replace_error_handler, True)
             offset = len(text)
             if len(self.text) != offset:
-                text, _ = str_decode_utf_8(self.text, len(self.text),
-                             'replace', errorhandler=replace_error_handler)
+                text, _ = str_decode_utf_8_impl(self.text, len(self.text),
+                             'replace', False, replace_error_handler, True)
             w_text = space.newtext(text)
         return space.newtuple([
             space.newtext(self.msg),
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -49,6 +49,7 @@
                                              space.newint(startingpos),
                                              space.newint(endingpos),
                                              space.newtext(msg)]))
+        return u'', None, 0
     return raise_unicode_exception_encode
 
 def default_error_encode(
@@ -106,8 +107,8 @@
                                     force_replace=False)
     elif _MACOSX:
         uni = space.utf8_w(w_uni)
-        bytes = runicode.unicode_encode_utf_8_impl(
-            uni, len(uni), 'surrogateescape',
+        bytes = unicodehelper.utf8_encode_utf_8(
+            uni, 'surrogateescape',
             errorhandler=state.encode_error_handler,
             allow_surrogates=False)
     elif space.sys.filesystemencoding is None or state.codec_need_encodings:
@@ -120,8 +121,7 @@
         uni = space.realunicode_w(w_uni)
         if u'\x00' in uni:
             raise oefmt(space.w_ValueError, "embedded null character")
-        bytes = unicode_encode_locale_surrogateescape(
-            uni, errorhandler=encode_error_handler(space))
+        bytes = unicode_encode_locale_surrogateescape(uni)
     else:
         from pypy.module.sys.interp_encoding import getfilesystemencoding
         return space.call_method(w_uni, 'encode',

From pypy.commits at gmail.com  Tue Aug  7 16:03:38 2018
From: pypy.commits at gmail.com (mattip)
Date: Tue, 07 Aug 2018 13:03:38 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: add partial implementation for
 interp_codecs
Message-ID: <5b69fb1a.1c69fb81.fc18d.9bae@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94971:a3b31b84b094
Date: 2018-08-07 09:48 -0700
http://bitbucket.org/pypy/pypy/changeset/a3b31b84b094/

Log:	add partial implementation for interp_codecs

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -225,8 +225,13 @@
     # cannot be ASCII, cannot have surrogates, I believe
     return res.build(), len(s), len(s)
 
-def utf8_encode_utf_8(s, errors, errorhandler):
-    # needed by tests
+def utf8_encode_utf_8(s, errors, errorhandler, allow_surrogates=False):
+    # XXX completly implement this
+    try:
+        lgt = rutf8.check_utf8(s, allow_surrogates=allow_surrogates)
+    except rutf8.CheckError as e:
+        s, lgt = errorhandler(errors, 'encoding',
+                    'surrogates not allowed', s, e.pos, e.pos + 1)
     return s
 
 def utf8_encode_latin_1(s, errors, errorhandler):

From pypy.commits at gmail.com  Tue Aug  7 17:11:37 2018
From: pypy.commits at gmail.com (arigo)
Date: Tue, 07 Aug 2018 14:11:37 -0700 (PDT)
Subject: [pypy-commit] pypy default: Add flag 'no_implicit_octal' to
 string_to_int()
Message-ID: <5b6a0b09.1c69fb81.c9521.c8a9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94972:746e52c25681
Date: 2018-08-07 23:06 +0200
http://bitbucket.org/pypy/pypy/changeset/746e52c25681/

Log:	Add flag 'no_implicit_octal' to string_to_int()

diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -868,7 +868,7 @@
 # String parsing support
 # ---------------------------
 
-def string_to_int(s, base=10, allow_underscores=False):
+def string_to_int(s, base=10, allow_underscores=False, no_implicit_octal=False):
     """Utility to converts a string to an integer.
     If base is 0, the proper base is guessed based on the leading
     characters of 's'.  Raises ParseStringError in case of error.
@@ -884,6 +884,9 @@
     while True:
         digit = p.next_digit()
         if digit == -1:
+            if no_implicit_octal:
+                if p.oldstyle_initial_zero and result != 0:
+                    p.error()
             return result
 
         if p.sign == -1:
diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -429,6 +429,7 @@
 
 # iterator-like class
 class NumberStringParser:
+    oldstyle_initial_zero = False
 
     def error(self):
         raise ParseStringError("invalid literal for %s() with base %d" %
@@ -445,7 +446,6 @@
         self.sign = sign
         self.original_base = base
         self.allow_underscores = allow_underscores
-        self.last_is_underscore = False
 
         if base == 0:
             if s.startswith('0x') or s.startswith('0X'):
@@ -453,6 +453,8 @@
             elif s.startswith('0b') or s.startswith('0B'):
                 base = 2
             elif s.startswith('0'): # also covers the '0o' case
+                if not (s.startswith('0o') or s.startswith('0O')):
+                    self.oldstyle_initial_zero = True
                 base = 8
             else:
                 base = 10
@@ -478,6 +480,11 @@
     def next_digit(self): # -1 => exhausted
         if self.i < self.n:
             c = self.s[self.i]
+            if self.allow_underscores and c == '_':
+                self.i += 1
+                if self.i >= self.n:
+                    self.error()
+                c = self.s[self.i]
             digit = ord(c)
             if '0' <= c <= '9':
                 digit -= ord('0')
@@ -485,22 +492,13 @@
                 digit = (digit - ord('A')) + 10
             elif 'a' <= c <= 'z':
                 digit = (digit - ord('a')) + 10
-            elif c == '_' and self.allow_underscores:
-                if self.last_is_underscore:
-                    self.error()
-                self.last_is_underscore = True
-                self.i += 1
-                return self.next_digit()
             else:
                 self.error()
             if digit >= self.base:
                 self.error()
             self.i += 1
-            self.last_is_underscore = False
             return digit
         else:
-            if self.last_is_underscore:
-                self.error()
             return -1
 
     def prev_digit(self):
diff --git a/rpython/rlib/test/test_rarithmetic.py b/rpython/rlib/test/test_rarithmetic.py
--- a/rpython/rlib/test/test_rarithmetic.py
+++ b/rpython/rlib/test/test_rarithmetic.py
@@ -591,13 +591,36 @@
         ]
         for x in VALID_UNDERSCORE_LITERALS:
             print x
-            y = string_to_int(x, base=0, allow_underscores=True)
+            y = string_to_int(x, base=0, allow_underscores=True,
+                              no_implicit_octal=True)
             assert y == int(x.replace('_', ''), base=0)
         for x in INVALID_UNDERSCORE_LITERALS:
             print x
             py.test.raises(ParseStringError, string_to_int, x, base=0,
                            allow_underscores=True)
 
+    def test_no_implicit_octal(self):
+        TESTS = ['00', '000', '00_00', '02', '0377', '02_34']
+        for x in TESTS:
+            for valid_underscore in [False, True]:
+                for no_implicit_octal in [False, True]:
+                    print x, valid_underscore, no_implicit_octal
+                    expected_ok = True
+                    if no_implicit_octal and any('1' <= c <= '7' for c in x):
+                        expected_ok = False
+                    if not valid_underscore and '_' in x:
+                        expected_ok = False
+                    if expected_ok:
+                        y = string_to_int(x, base=0,
+                                          allow_underscores=valid_underscore,
+                                          no_implicit_octal=no_implicit_octal)
+                        assert y == int(x.replace('_', ''), base=8)
+                    else:
+                        py.test.raises(ParseStringError, string_to_int, x,
+                                       base=0,
+                                       allow_underscores=valid_underscore,
+                                       no_implicit_octal=no_implicit_octal)
+
 
 class TestExplicitIntsizes:
 

From pypy.commits at gmail.com  Tue Aug  7 17:11:39 2018
From: pypy.commits at gmail.com (arigo)
Date: Tue, 07 Aug 2018 14:11:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5b6a0b0b.1c69fb81.b6e30.a856@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94973:b9da0b002569
Date: 2018-08-07 23:10 +0200
http://bitbucket.org/pypy/pypy/changeset/b9da0b002569/

Log:	hg merge default

diff too long, truncating to 2000 out of 2739 lines

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -7,7 +7,8 @@
 
 .. branch: cppyy-packaging
 
-Upgrade to backend 1.2.0, improved handling of templated methods and
+Main items: vastly better template resolution and improved performance. In
+detail: upgrade to backend 1.4, improved handling of templated methods and
 functions (in particular automatic deduction of types), improved pythonization
 interface, range of compatibility fixes for Python3, free functions now take
 fast libffi path when possible, moves for strings (incl. from Python str),
diff --git a/pypy/module/_cffi_backend/errorbox.py b/pypy/module/_cffi_backend/errorbox.py
--- a/pypy/module/_cffi_backend/errorbox.py
+++ b/pypy/module/_cffi_backend/errorbox.py
@@ -69,7 +69,10 @@
                 import sys
                 class FileLike:
                     def write(self, x):
-                        of.write(x)
+                        try:
+                            of.write(x)
+                        except:
+                            pass
                         self.buf += x
                 fl = FileLike()
                 fl.buf = ''
diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py
--- a/pypy/module/_cppyy/capi/loadable_capi.py
+++ b/pypy/module/_cppyy/capi/loadable_capi.py
@@ -1,13 +1,18 @@
 import os
+
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib import jit, jit_libffi, libffi, rdynload, objectmodel
 from rpython.rlib.rarithmetic import r_singlefloat
 from rpython.tool import leakfinder
 
-from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.error import oefmt
+from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.argument import Arguments
+from pypy.interpreter.gateway import interp2app, interpindirect2app
+from pypy.interpreter.typedef import TypeDef
+from pypy.objspace.std.iterobject import W_AbstractSeqIterObject
 
+from pypy.module._rawffi.array import W_ArrayInstance
 from pypy.module._cffi_backend import ctypefunc, ctypeprim, cdataobj, misc
 from pypy.module._cffi_backend import newtype
 from pypy.module._cppyy import ffitypes
@@ -23,10 +28,11 @@
 
 class _Arg:         # poor man's union
     _immutable_ = True
-    def __init__(self, tc, h = 0, l = -1, s = '', p = rffi.cast(rffi.VOIDP, 0)):
+    def __init__(self, tc, h = 0, l = -1, d = -1., s = '', p = rffi.cast(rffi.VOIDP, 0)):
         self.tc      = tc
         self._handle = h
         self._long   = l
+        self._double = d
         self._string = s
         self._voidp  = p
 
@@ -40,6 +46,11 @@
     def __init__(self, val):
         _Arg.__init__(self, 'l', l = val)
 
+class _ArgD(_Arg):
+    _immutable_ = True
+    def __init__(self, val):
+        _Arg.__init__(self, 'd', d = val)
+
 class _ArgS(_Arg):
     _immutable_ = True
     def __init__(self, val):
@@ -89,6 +100,9 @@
                     assert obj._voidp != rffi.cast(rffi.VOIDP, 0)
                     data = rffi.cast(rffi.VOIDPP, data)
                     data[0] = obj._voidp
+                elif obj.tc == 'd':
+                    assert isinstance(argtype, ctypeprim.W_CTypePrimitiveFloat)
+                    misc.write_raw_float_data(data, rffi.cast(rffi.DOUBLE, obj._double), argtype.size)
                 else:    # only other use is string
                     assert obj.tc == 's'
                     n = len(obj._string)
@@ -182,6 +196,7 @@
             'call_f'       : ([c_method, c_object, c_int, c_voidp],   c_float),
             'call_d'       : ([c_method, c_object, c_int, c_voidp],   c_double),
             'call_ld'      : ([c_method, c_object, c_int, c_voidp],   c_ldouble),
+            'call_nld'     : ([c_method, c_object, c_int, c_voidp],   c_double),
 
             'call_r'       : ([c_method, c_object, c_int, c_voidp],   c_voidp),
             # call_s actually takes an size_t* as last parameter, but this will do
@@ -236,6 +251,8 @@
             'method_prototype'         : ([c_scope, c_method, c_int], c_ccharp),
             'is_const_method'          : ([c_method],                 c_int),
 
+            'get_num_templated_methods': ([c_scope],                  c_int),
+            'get_templated_method_name': ([c_scope, c_index],         c_ccharp),
             'exists_method_template'   : ([c_scope, c_ccharp],        c_int),
             'method_is_template'       : ([c_scope, c_index],         c_int),
             'get_method_template'      : ([c_scope, c_ccharp, c_ccharp],        c_method),
@@ -272,9 +289,11 @@
             'stdstring2charp'          : ([c_object, c_voidp],        c_ccharp),
             'stdstring2stdstring'      : ([c_object],                 c_object),
 
-            'stdvector_valuetype'      : ([c_ccharp],                 c_ccharp),
-            'stdvector_valuesize'      : ([c_ccharp],                 c_size_t),
+            'longdouble2double'        : ([c_voidp],                  c_double),
+            'double2longdouble'        : ([c_double, c_voidp],        c_void),
 
+            'vectorbool_getitem'       : ([c_object, c_int],          c_int),
+            'vectorbool_setitem'       : ([c_object, c_int, c_int],   c_void),
         }
 
         # size/offset are backend-specific but fixed after load
@@ -401,7 +420,9 @@
     return rffi.cast(rffi.DOUBLE, space.float_w(call_capi(space, 'call_d', args)))
 def c_call_ld(space, cppmethod, cppobject, nargs, cargs):
     args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs)]
-    return rffi.cast(rffi.LONGDOUBLE, space.float_w(call_capi(space, 'call_ld', args)))
+    #return rffi.cast(rffi.LONGDOUBLE, space.float_w(call_capi(space, 'call_ld', args)))
+    # call_nld narrows long double to double
+    return rffi.cast(rffi.DOUBLE, space.float_w(call_capi(space, 'call_nld', args)))
 
 def c_call_r(space, cppmethod, cppobject, nargs, cargs):
     args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs)]
@@ -561,16 +582,21 @@
 def c_is_const_method(space, cppmeth):
     return space.bool_w(call_capi(space, 'is_const_method', [_ArgH(cppmeth)]))
 
+def c_get_num_templated_methods(space, cppscope):
+    return space.int_w(call_capi(space, 'method_is_template', [_ArgH(cppscope.handle)]))
+def c_get_templated_method_name(space, cppscope, index):
+    args = [_ArgH(cppscope.handle), _ArgL(index)]
+    return charp2str_free(space, call_capi(space, 'method_is_template', args))
 def c_exists_method_template(space, cppscope, name):
     args = [_ArgH(cppscope.handle), _ArgS(name)]
     return space.bool_w(call_capi(space, 'exists_method_template', args))
 def c_method_is_template(space, cppscope, index):
     args = [_ArgH(cppscope.handle), _ArgL(index)]
     return space.bool_w(call_capi(space, 'method_is_template', args))
-
 def c_get_method_template(space, cppscope, name, proto):
     args = [_ArgH(cppscope.handle), _ArgS(name), _ArgS(proto)]
     return rffi.cast(C_METHOD, space.uint_w(call_capi(space, 'get_method_template', args)))
+
 def c_get_global_operator(space, nss, lc, rc, op):
     if nss is not None:
         args = [_ArgH(nss.handle), _ArgH(lc.handle), _ArgH(rc.handle), _ArgS(op)]
@@ -619,7 +645,7 @@
     return space.bool_w(call_capi(space, 'is_enum_data', args))
 def c_get_dimension_size(space, cppscope, datamember_index, dim_idx):
     args = [_ArgH(cppscope.handle), _ArgL(datamember_index), _ArgL(dim_idx)]
-    return space.bool_w(call_capi(space, 'get_dimension_size', args))
+    return space.int_w(call_capi(space, 'get_dimension_size', args))
 
 # misc helpers ---------------------------------------------------------------
 def c_strtoll(space, svalue):
@@ -650,24 +676,94 @@
 def c_stdstring2stdstring(space, cppobject):
     return _cdata_to_cobject(space, call_capi(space, 'stdstring2stdstring', [_ArgH(cppobject)]))
 
-def c_stdvector_valuetype(space, pystr):
-    return charp2str_free(space, call_capi(space, 'stdvector_valuetype', [_ArgS(pystr)]))
+def c_longdouble2double(space, addr):
+    return space.float_w(call_capi(space, 'longdouble2double', [_ArgP(addr)]))
+def c_double2longdouble(space, dval, addr):
+    call_capi(space, 'double2longdouble', [_ArgD(dval), _ArgP(addr)])
 
-def c_stdvector_valuetype(space, pystr):
-    return charp2str_free(space, call_capi(space, 'stdvector_valuetype', [_ArgS(pystr)]))
-def c_stdvector_valuesize(space, pystr):
-    return _cdata_to_size_t(space, call_capi(space, 'stdvector_valuesize', [_ArgS(pystr)]))
+def c_vectorbool_getitem(space, vbool, idx):
+    return call_capi(space, 'vectorbool_getitem', [_ArgH(vbool), _ArgL(idx)])
+def c_vectorbool_setitem(space, vbool, idx, value):
+    call_capi(space, 'vectorbool_setitem', [_ArgH(vbool), _ArgL(idx), _ArgL(value)])
 
 
 # TODO: factor these out ...
 # pythonizations
 def stdstring_c_str(space, w_self):
     """Return a python string taking into account \0"""
-
     from pypy.module._cppyy import interp_cppyy
     cppstr = space.interp_w(interp_cppyy.W_CPPInstance, w_self, can_be_None=False)
     return space.newtext(c_stdstring2charp(space, cppstr._rawobject))
 
+def vbool_getindex(space, w_vbool, w_idx):
+    idx = space.getindex_w(w_idx, space.w_IndexError, "std::vector<bool> index")
+    sz = space.len_w(w_vbool)
+    if idx < 0: idx += sz
+    if idx < 0 or idx >= sz:
+        raise IndexError
+    return idx
+
+def vectorbool_getitem(space, w_self, w_idx):
+    """Index a std::vector<bool>, return the value"""
+    from pypy.module._cppyy import interp_cppyy
+    vbool = space.interp_w(interp_cppyy.W_CPPInstance, w_self, can_be_None=False)
+    idx = vbool_getindex(space, w_self, w_idx)
+    item = c_vectorbool_getitem(space, vbool._rawobject, idx)
+    return space.newbool(space.is_true(item))
+
+def vectorbool_setitem(space, w_self, w_idx, w_value):
+    """Index a std::vector<bool>, set the value"""
+    from pypy.module._cppyy import interp_cppyy
+    vbool = space.interp_w(interp_cppyy.W_CPPInstance, w_self, can_be_None=False)
+    idx = vbool_getindex(space, w_self, w_idx)
+    c_vectorbool_setitem(space, vbool._rawobject, idx, int(space.is_true(w_value)))
+
+class W_STLVectorIter(W_AbstractSeqIterObject):
+    # w_seq and index are in base class
+    _immutable_fields_ = ['converter', 'data', 'len', 'stride']
+
+    def __init__(self, space, w_vector):
+        W_AbstractSeqIterObject.__init__(self, w_vector)
+        # TODO: this should live in rpythonize.py or something so that the
+        # imports can move to the top w/o getting circles
+        from pypy.module._cppyy import interp_cppyy
+        assert isinstance(w_vector, interp_cppyy.W_CPPInstance)
+        vector = space.interp_w(interp_cppyy.W_CPPInstance, w_vector)
+
+        v_type = c_resolve_name(space, vector.clsdecl.name+'::value_type')
+        v_size = c_size_of_type(space, v_type)
+
+        if not v_type or not v_size:
+            raise NotImplementedError   # fallback on getitem
+
+        from pypy.module._cppyy import converter
+        self.converter = converter.get_converter(space, v_type, '')
+
+        # this 'data' is from the decl, so not the pythonized data from pythonify.py
+        w_arr = space.call_obj_args(vector.clsdecl.get_overload('data'), w_vector, Arguments(space, []))
+        arr = space.interp_w(W_ArrayInstance, w_arr, can_be_None=True)
+        if not arr:
+            raise OperationError(space.w_StopIteration, space.w_None)
+
+        self.data    = rffi.cast(rffi.CCHARP, space.uint_w(arr.getbuffer(space)))
+        self.len     = space.uint_w(space.call_obj_args(vector.clsdecl.get_overload('size'), w_vector, Arguments(space, [])))
+        self.stride  = v_size
+
+    def descr_next(self, space):
+        if self.w_seq is None:
+            raise OperationError(space.w_StopIteration, space.w_None)
+        if self.len <= self.index:
+            self.w_seq = None
+            raise OperationError(space.w_StopIteration, space.w_None)
+        offset = lltype.direct_ptradd(self.data, rffi.cast(rffi.SIZE_T, self.index*self.stride))
+        w_item = self.converter.from_memory(space, space.w_None, rffi.cast(rffi.LONG, offset))
+        self.index += 1
+        return w_item
+
+def stdvector_iter(space, w_self):
+    return W_STLVectorIter(space, w_self)
+
+
 # setup pythonizations for later use at run-time
 _pythonizations = {}
 def register_pythonizations(space):
@@ -678,6 +774,12 @@
         ### std::string
         stdstring_c_str,
 
+        ### std::vector
+        stdvector_iter,
+
+        ### std::vector<bool>
+        vectorbool_getitem,
+        vectorbool_setitem,
     ]
 
     for f in allfuncs:
@@ -692,3 +794,10 @@
         space.setattr(w_pycppclass, space.newtext("c_str"), _pythonizations["stdstring_c_str"])
         _method_alias(space, w_pycppclass, "_cppyy_as_builtin", "c_str")
         _method_alias(space, w_pycppclass, "__str__",           "c_str")
+
+    if  name.find("std::vector<bool", 0, 16) == 0:
+        space.setattr(w_pycppclass, space.newtext("__getitem__"), _pythonizations["vectorbool_getitem"])
+        space.setattr(w_pycppclass, space.newtext("__setitem__"), _pythonizations["vectorbool_setitem"])
+
+    elif name.find("std::vector", 0, 11) == 0:
+        space.setattr(w_pycppclass, space.newtext("__iter__"), _pythonizations["stdvector_iter"])
diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -115,7 +115,7 @@
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         self._is_abstract(space)
 
     def to_memory(self, space, w_obj, w_value, offset):
@@ -142,7 +142,7 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         # read access, so no copy needed
         address_value = self._get_raw_address(space, w_obj, offset)
         address = rffi.cast(rffi.ULONG, address_value)
@@ -184,7 +184,7 @@
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'o'
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         # read access, so no copy needed
         address_value = self._get_raw_address(space, w_obj, offset)
         address = rffi.cast(rffi.ULONGP, address_value)
@@ -211,10 +211,13 @@
         x[0] = self._unwrap_object(space, w_obj)
 
     def default_argument_libffi(self, space, address):
+        if not self.valid_default:
+            from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
+            raise FastCallNotPossible
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self.default
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = self._get_raw_address(space, w_obj, offset)
         rffiptr = rffi.cast(self.c_ptrtype, address)
         return self._wrap_object(space, rffiptr[0])
@@ -224,7 +227,7 @@
         rffiptr = rffi.cast(self.c_ptrtype, address)
         rffiptr[0] = self._unwrap_object(space, w_value)
 
-class ConstRefNumericTypeConverterMixin(NumericTypeConverterMixin):
+class ConstRefNumericTypeConverterMixin(object):
     _mixin_ = True
 
     def cffi_type(self, space):
@@ -283,7 +286,7 @@
         x = rffi.cast(rffi.LONGP, address)
         x[0] = self._unwrap_object(space, w_obj)
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = rffi.cast(rffi.CCHARP, self._get_raw_address(space, w_obj, offset))
         if address[0] == '\x01':
             return space.w_True
@@ -308,7 +311,7 @@
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = rffi.cast(rffi.CCHARP, self._get_raw_address(space, w_obj, offset))
         return space.newbytes(address[0])
 
@@ -321,59 +324,92 @@
     pass
 
 class FloatConverter(ffitypes.typeid(rffi.FLOAT), FloatTypeConverterMixin, TypeConverter):
-    _immutable_fields_ = ['default']
+    _immutable_fields_ = ['default', 'valid_default']
 
     def __init__(self, space, default):
-        if default:
+        self.valid_default = False
+        try:
             fval = float(rfloat.rstring_to_float(default))
-        else:
+            self.valid_default = True
+        except Exception:
             fval = float(0.)
-        self.default = r_singlefloat(fval)
+        self.default = rffi.cast(rffi.FLOAT, r_singlefloat(fval))
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = self._get_raw_address(space, w_obj, offset)
         rffiptr = rffi.cast(self.c_ptrtype, address)
         return self._wrap_object(space, rffiptr[0])
 
-class ConstFloatRefConverter(FloatConverter):
+class ConstFloatRefConverter(ConstRefNumericTypeConverterMixin, FloatConverter):
     _immutable_fields_ = ['typecode']
     typecode = 'f'
 
-    def cffi_type(self, space):
-        state = space.fromcache(ffitypes.State)
-        return state.c_voidp
-
-    def convert_argument_libffi(self, space, w_obj, address, scratch):
-        from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
-        raise FastCallNotPossible
-
 class DoubleConverter(ffitypes.typeid(rffi.DOUBLE), FloatTypeConverterMixin, TypeConverter):
-    _immutable_fields_ = ['default']
+    _immutable_fields_ = ['default', 'valid_default']
 
     def __init__(self, space, default):
-        if default:
+        self.valid_default = False
+        try:
             self.default = rffi.cast(self.c_type, rfloat.rstring_to_float(default))
-        else:
+            self.valid_default = True
+        except Exception:
             self.default = rffi.cast(self.c_type, 0.)
 
 class ConstDoubleRefConverter(ConstRefNumericTypeConverterMixin, DoubleConverter):
     _immutable_fields_ = ['typecode']
     typecode = 'd'
 
-class LongDoubleConverter(ffitypes.typeid(rffi.LONGDOUBLE), FloatTypeConverterMixin, TypeConverter):
-    _immutable_fields_ = ['default']
+class LongDoubleConverter(TypeConverter):
+    _immutable_fields_ = ['default', 'valid_default']
+    typecode = 'g'
 
     def __init__(self, space, default):
-        if default:
-            fval = float(rfloat.rstring_to_float(default))
-        else:
-            fval = float(0.)
-        self.default = r_longfloat(fval)
+        self.valid_default = False
+        try:
+            # use float() instead of cast with r_longfloat
+            fval = rffi.cast(rffi.DOUBLE, rfloat.rstring_to_float(default))
+            self.valid_default = True
+        except Exception:
+            fval = rffi.cast(rffi.DOUBLE, 0.)
+        #self.default = r_longfloat(fval)
+        self.default = fval
+
+    def convert_argument(self, space, w_obj, address):
+        x = rffi.cast(rffi.VOIDP, address)
+        capi.c_double2longdouble(space, space.float_w(w_obj), x)
+        ba = rffi.cast(rffi.CCHARP, address)
+        ba[capi.c_function_arg_typeoffset(space)] = self.typecode
+
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
+        x = rffi.cast(rffi.VOIDP, address)
+        capi.c_double2longdouble(space, space.float_w(w_obj), x)
+
+    def default_argument_libffi(self, space, address):
+        if not self.valid_default:
+            from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
+            raise FastCallNotPossible
+        x = rffi.cast(rffi.VOIDP, address)
+        capi.c_double2longdouble(space, self.default, x)
+
+    def from_memory(self, space, w_obj, offset):
+        address = self._get_raw_address(space, w_obj, offset)
+        rffiptr = rffi.cast(rffi.VOIDP, address)
+        return space.newfloat(capi.c_longdouble2double(space, rffiptr))
+
+    def to_memory(self, space, w_obj, w_value, offset):
+        address = self._get_raw_address(space, w_obj, offset)
+        rffiptr = rffi.cast(rffi.VOIDP, address)
+        capi.c_double2longdouble(space, space.float_w(w_value), rffiptr)
 
 class ConstLongDoubleRefConverter(ConstRefNumericTypeConverterMixin, LongDoubleConverter):
     _immutable_fields_ = ['typecode']
     typecode = 'g'
 
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
+        capi.c_double2longdouble(space, space.float_w(w_obj), rffi.cast(rffi.VOIDP, scratch))
+        x = rffi.cast(rffi.VOIDPP, address)
+        x[0] = scratch
+
 
 class CStringConverter(TypeConverter):
     def convert_argument(self, space, w_obj, address):
@@ -383,7 +419,7 @@
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'p'
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = self._get_raw_address(space, w_obj, offset)
         charpptr = rffi.cast(rffi.CCHARPP, address)
         return space.newtext(rffi.charp2str(charpptr[0]))
@@ -397,13 +433,15 @@
     def __init__(self, space, extra):
         self.size = extra
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = self._get_raw_address(space, w_obj, offset)
         charpptr = rffi.cast(rffi.CCHARP, address)
-        strsize = self.size
-        if charpptr[self.size-1] == '\0':
-            strsize = self.size-1  # rffi will add \0 back
-        return space.newtext(rffi.charpsize2str(charpptr, strsize))
+        if 0 <= self.size and self.size != 2**31-1:   # cling's code for "unknown" (?)
+            strsize = self.size
+            if charpptr[self.size-1] == '\0':
+                strsize = self.size-1  # rffi will add \0 back
+            return space.newtext(rffi.charpsize2str(charpptr, strsize))
+        return space.newtext(rffi.charp2str(charpptr))
 
 
 class VoidPtrConverter(TypeConverter):
@@ -428,7 +466,7 @@
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = self._unwrap_object(space, w_obj)
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         # returned as a long value for the address (INTPTR_T is not proper
         # per se, but rffi does not come with a PTRDIFF_T)
         address = self._get_raw_address(space, w_obj, offset)
@@ -540,7 +578,7 @@
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible       # TODO: by-value is a jit_libffi special case
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
         from pypy.module._cppyy import interp_cppyy
         return interp_cppyy.wrap_cppinstance(space, address, self.clsdecl, do_cast=False)
@@ -560,7 +598,7 @@
                 return capi.C_NULL_OBJECT
             raise e
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
         from pypy.module._cppyy import interp_cppyy
         return interp_cppyy.wrap_cppinstance(space, address, self.clsdecl, do_cast=False)
@@ -586,7 +624,7 @@
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
         from pypy.module._cppyy import interp_cppyy
         return interp_cppyy.wrap_cppinstance(
@@ -620,6 +658,26 @@
             lltype.free(self.ref_buffer, flavor='raw')
             self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
 
+class InstanceArrayConverter(InstancePtrConverter):
+    _immutable_fields_ = ['size']
+
+    def __init__(self, space, clsdecl, array_size, dimensions):
+        InstancePtrConverter.__init__(self, space, clsdecl)
+        if array_size <= 0 or array_size == 2**31-1:   # cling's code for "unknown" (?)
+            self.size = sys.maxint
+        else:
+            self.size = array_size
+        # peel one off as that should be the same as the array size
+        self.dimensions = dimensions[1:]
+
+    def from_memory(self, space, w_obj, offset):
+        address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
+        return lowlevelviews.W_ArrayOfInstances(space, self.clsdecl, address, self.size, self.dimensions)
+
+    def to_memory(self, space, w_obj, w_value, offset):
+        self._is_abstract(space)
+
+
 class StdStringConverter(InstanceConverter):
     def __init__(self, space, extra):
         from pypy.module._cppyy import interp_cppyy
@@ -790,7 +848,7 @@
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = self.typecode
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
         from pypy.module._cppyy import interp_cppyy
         return interp_cppyy.wrap_cppinstance(space, address,
@@ -799,7 +857,7 @@
 class SmartPtrPtrConverter(SmartPtrConverter):
     typecode    = 'o'
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         self._is_abstract(space)
 
     def to_memory(self, space, w_obj, w_value, offset):
@@ -811,7 +869,7 @@
 
 
 class MacroConverter(TypeConverter):
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         # TODO: get the actual type info from somewhere ...
         address = self._get_raw_address(space, w_obj, offset)
         longptr = rffi.cast(rffi.LONGP, address)
@@ -844,20 +902,28 @@
         pass
 
     # match of decorated, unqualified type
-    compound = helper.compound(name)
+    cpd = helper.compound(name)
     clean_name = capi.c_resolve_name(space, helper.clean_type(name))
     try:
-        return _converters[clean_name+compound](space, default)
+        return _converters[clean_name+cpd](space, default)
     except KeyError:
         pass
 
-    # arrays
+    # arrays (array_size may be negative, meaning: no size or no size found)
+    array_size = -1
+    if cpd == "[]":
+        array_size = helper.array_size(_name)    # uses original arg
+    elif cpd == '*' and ':' in default:
+        # this happens for multi-dimensional arrays: those are described as pointers
+        cpd = "[]"
+        splitpos = default.find(':')
+        if 0 < splitpos:     # always true, but needed for annotator
+            array_size = int(default[:splitpos])
+
     try:
-        # array_index may be negative to indicate no size or no size found
-        array_size = helper.array_size(_name)     # uses original arg
         # TODO: using clean_name here drops const (e.g. const char[] will
         # never be seen this way)
-        return _a_converters[clean_name+compound](space, array_size)
+        return _a_converters[clean_name+cpd](space, array_size)
     except KeyError:
         pass
 
@@ -871,24 +937,28 @@
         # check smart pointer type
         check_smart = capi.c_smartptr_info(space, clean_name)
         if check_smart[0]:
-            if compound == '':
+            if cpd == '':
                 return SmartPtrConverter(space, clsdecl, check_smart[1], check_smart[2])
-            elif compound == '*':
+            elif cpd == '*':
                 return SmartPtrPtrConverter(space, clsdecl, check_smart[1], check_smart[2])
-            elif compound == '&':
+            elif cpd == '&':
                 return SmartPtrRefConverter(space, clsdecl, check_smart[1], check_smart[2])
             # fall through: can still return smart pointer in non-smart way
 
         # type check for the benefit of the annotator
-        if compound == "*":
+        if cpd == "*":
             return InstancePtrConverter(space, clsdecl)
-        elif compound == "&":
+        elif cpd == "&":
             return InstanceRefConverter(space, clsdecl)
-        elif compound == "&&":
+        elif cpd == "&&":
             return InstanceMoveConverter(space, clsdecl)
-        elif compound == "**":
+        elif cpd in ["**", "*[]", "&*"]:
             return InstancePtrPtrConverter(space, clsdecl)
-        elif compound == "":
+        elif cpd == "[]" and array_size > 0:
+            # default encodes the dimensions
+            dims = default.split(':')
+            return InstanceArrayConverter(space, clsdecl, array_size, dims)
+        elif cpd == "":
             return InstanceConverter(space, clsdecl)
     elif "(anonymous)" in name:
         # special case: enum w/o a type name
@@ -900,7 +970,7 @@
             return FunctionPointerConverter(space, name[pos+2:])
 
     # void* or void converter (which fails on use)
-    if 0 <= compound.find('*'):
+    if 0 <= cpd.find('*'):
         return VoidPtrConverter(space, default)  # "user knows best"
 
     # return a void converter here, so that the class can be build even
@@ -915,8 +985,8 @@
 _converters["const float&"]             = ConstFloatRefConverter
 _converters["double"]                   = DoubleConverter
 _converters["const double&"]            = ConstDoubleRefConverter
-#_converters["long double"]              = LongDoubleConverter
-#_converters["const long double&"]       = ConstLongDoubleRefConverter
+_converters["long double"]              = LongDoubleConverter
+_converters["const long double&"]       = ConstLongDoubleRefConverter
 _converters["const char*"]              = CStringConverter
 _converters["void*"]                    = VoidPtrConverter
 _converters["void**"]                   = VoidPtrPtrConverter
@@ -950,7 +1020,12 @@
             _immutable_ = True
             typecode = c_tc
             def __init__(self, space, default):
-                self.default = rffi.cast(self.c_type, capi.c_strtoll(space, default))
+                self.valid_default = False
+                try:
+                    self.default = rffi.cast(self.c_type, capi.c_strtoll(space, default))
+                    self.valid_default = True
+                except Exception:
+                    self.default = rffi.cast(self.c_type, 0)
         class ConstRefConverter(ConstRefNumericTypeConverterMixin, BasicConverter):
             _immutable_ = True
         for name in names:
@@ -967,7 +1042,12 @@
             _immutable_ = True
             typecode = c_tc
             def __init__(self, space, default):
-                self.default = rffi.cast(self.c_type, capi.c_strtoll(space, default))
+                self.valid_default = False
+                try:
+                    self.default = rffi.cast(self.c_type, capi.c_strtoll(space, default))
+                    self.valid_default = True
+                except Exception:
+                    self.default = rffi.cast(self.c_type, 0)
         class ConstRefConverter(ConstRefNumericTypeConverterMixin, BasicConverter):
             _immutable_ = True
         for name in names:
@@ -987,7 +1067,12 @@
             _immutable_ = True
             typecode = c_tc
             def __init__(self, space, default):
-                self.default = rffi.cast(self.c_type, capi.c_strtoull(space, default))
+                self.valid_default = False
+                try:
+                    self.default = rffi.cast(self.c_type, capi.c_strtoull(space, default))
+                    self.valid_default = True
+                except Exception:
+                    self.default = rffi.cast(self.c_type, 0)
         class ConstRefConverter(ConstRefNumericTypeConverterMixin, BasicConverter):
             _immutable_ = True
         for name in names:
diff --git a/pypy/module/_cppyy/executor.py b/pypy/module/_cppyy/executor.py
--- a/pypy/module/_cppyy/executor.py
+++ b/pypy/module/_cppyy/executor.py
@@ -76,9 +76,6 @@
 class NumericExecutorMixin(object):
     _mixin_ = True
 
-    #def _wrap_object(self, space, obj):
-    #    return getattr(space, self.wrapper)(obj)
-
     def execute(self, space, cppmethod, cppthis, num_args, args):
         result = self.c_stubcall(space, cppmethod, cppthis, num_args, args)
         return self._wrap_object(space, rffi.cast(self.c_type, result))
@@ -100,13 +97,10 @@
         self.w_item = w_item
         self.do_assign = True
 
-    #def _wrap_object(self, space, obj):
-    #    return getattr(space, self.wrapper)(rffi.cast(self.c_type, obj))
-
     def _wrap_reference(self, space, rffiptr):
         if self.do_assign:
             rffiptr[0] = rffi.cast(self.c_type, self._unwrap_object(space, self.w_item))
-        self.do_assign = False
+            self.do_assign = False
         return self._wrap_object(space, rffiptr[0])    # all paths, for rtyper
 
     def execute(self, space, cppmethod, cppthis, num_args, args):
@@ -119,6 +113,48 @@
         return self._wrap_reference(space,
             rffi.cast(self.c_ptrtype, rffi.cast(rffi.VOIDPP, result)[0]))
 
+class LongDoubleExecutorMixin(object):
+    # Note: not really supported, but returns normal double
+    _mixin_ = True
+
+    def execute(self, space, cppmethod, cppthis, num_args, args):
+        result = self.c_stubcall(space, cppmethod, cppthis, num_args, args)
+        return space.newfloat(result)
+
+    def execute_libffi(self, space, cif_descr, funcaddr, buffer):
+        from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
+        raise FastCallNotPossible
+
+class LongDoubleExecutor(ffitypes.typeid(rffi.LONGDOUBLE), LongDoubleExecutorMixin, Executor):
+    _immutable_ = True
+    c_stubcall  = staticmethod(capi.c_call_ld)
+
+class LongDoubleRefExecutorMixin(NumericRefExecutorMixin):
+    # Note: not really supported, but returns normal double
+    _mixin_ = True
+
+    def _wrap_reference(self, space, rffiptr):
+        if self.do_assign:
+            capi.c_double2longdouble(space, space.float_w(self.w_item), rffiptr)
+            self.do_assign = False
+            return self.w_item
+        return space.newfloat(capi.c_longdouble2double(space, rffiptr))
+
+    def execute(self, space, cppmethod, cppthis, num_args, args):
+        result = capi.c_call_r(space, cppmethod, cppthis, num_args, args)
+        return self._wrap_reference(space, rffi.cast(self.c_ptrtype, result))
+
+    def execute_libffi(self, space, cif_descr, funcaddr, buffer):
+        jit_libffi.jit_ffi_call(cif_descr, funcaddr, buffer)
+        result = rffi.ptradd(buffer, cif_descr.exchange_result)
+        return self._wrap_reference(space,
+            rffi.cast(self.c_ptrtype, rffi.cast(rffi.VOIDPP, result)[0]))
+
+class LongDoubleRefExecutor(ffitypes.typeid(rffi.LONGDOUBLE), LongDoubleRefExecutorMixin, Executor):
+    def cffi_type(self, space):
+        state = space.fromcache(ffitypes.State)
+        return state.c_voidp
+
 
 class CStringExecutor(Executor):
     def execute(self, space, cppmethod, cppthis, num_args, args):
@@ -345,6 +381,10 @@
 _executors["void*"]               = PtrTypeExecutor
 _executors["const char*"]         = CStringExecutor
 
+# long double not really supported: narrows to double
+_executors["long double"]          = LongDoubleExecutor
+_executors["long double&"]         = LongDoubleRefExecutor
+
 # special cases (note: 'string' aliases added below)
 _executors["constructor"]         = ConstructorExecutor
 
diff --git a/pypy/module/_cppyy/ffitypes.py b/pypy/module/_cppyy/ffitypes.py
--- a/pypy/module/_cppyy/ffitypes.py
+++ b/pypy/module/_cppyy/ffitypes.py
@@ -296,7 +296,8 @@
     _immutable_fields_ = ['c_type', 'c_ptrtype', 'typecode']
 
     c_type      = rffi.LONGDOUBLE
-    c_ptrtype   = rffi.LONGDOUBLEP
+    # c_ptrtype   = rffi.LONGDOUBLEP   # useless type at this point
+    c_ptrtype   = rffi.VOIDP
     typecode    = 'g'
 
     # long double is not really supported ...
@@ -304,7 +305,7 @@
         return r_longfloat(space.float_w(w_obj))
 
     def _wrap_object(self, space, obj):
-        return space.wrap(obj)
+        return space.newfloat(obj)
 
     def cffi_type(self, space):
         state = space.fromcache(State)
diff --git a/pypy/module/_cppyy/helper.py b/pypy/module/_cppyy/helper.py
--- a/pypy/module/_cppyy/helper.py
+++ b/pypy/module/_cppyy/helper.py
@@ -117,16 +117,10 @@
     # TODO: perhaps absorb or "pythonify" these operators?
     return cppname
 
-if sys.hexversion < 0x3000000:
-    CPPYY__div__  = "__div__"
-    CPPYY__idiv__ = "__idiv__"
-    CPPYY__long__ = "__long__"
-    CPPYY__bool__ = "__nonzero__"
-else:
-    CPPYY__div__  = "__truediv__"
-    CPPYY__idiv__ = "__itruediv__"
-    CPPYY__long__ = "__int__"
-    CPPYY__bool__ = "__bool__"
+CPPYY__div__  = "__div__"
+CPPYY__idiv__ = "__idiv__"
+CPPYY__long__ = "__long__"
+CPPYY__bool__ = "__nonzero__"
 
 # _operator_mappings["[]"]  = "__setitem__"      # depends on return type
 # _operator_mappings["+"]   = "__add__"          # depends on # of args (see __pos__)
diff --git a/pypy/module/_cppyy/include/capi.h b/pypy/module/_cppyy/include/capi.h
--- a/pypy/module/_cppyy/include/capi.h
+++ b/pypy/module/_cppyy/include/capi.h
@@ -63,6 +63,8 @@
     double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
     RPY_EXTERN
     long double cppyy_call_ld(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
+    double cppyy_call_nld(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
 
     RPY_EXTERN
     void* cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
@@ -151,11 +153,15 @@
     RPY_EXTERN
     char* cppyy_method_signature(cppyy_method_t, int show_formalargs);
     RPY_EXTERN
-    char* cppyy_method_prototype(cppyy_scope_t scope, cppyy_method_t idx, int show_formalargs);
+    char* cppyy_method_prototype(cppyy_scope_t scope, cppyy_method_t, int show_formalargs);
     RPY_EXTERN
     int cppyy_is_const_method(cppyy_method_t);
 
     RPY_EXTERN
+    int get_num_templated_methods(cppyy_scope_t scope);
+    RPY_EXTERN
+    char* get_templated_method_name(cppyy_scope_t scope, cppyy_index_t imeth);
+    RPY_EXTERN
     int cppyy_exists_method_template(cppyy_scope_t scope, const char* name);
     RPY_EXTERN
     int cppyy_method_is_template(cppyy_scope_t scope, cppyy_index_t idx);
@@ -216,9 +222,14 @@
     cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr);
 
     RPY_EXTERN
-    const char* cppyy_stdvector_valuetype(const char* clname);
+    double cppyy_longdouble2double(void*);
     RPY_EXTERN
-    size_t      cppyy_stdvector_valuesize(const char* clname);
+    void   cppyy_double2longdouble(double, void*);
+
+    RPY_EXTERN
+    int         cppyy_vectorbool_getitem(cppyy_object_t ptr, int idx);
+    RPY_EXTERN
+    void        cppyy_vectorbool_setitem(cppyy_object_t ptr, int idx, int value);
 
 #ifdef __cplusplus
 }
diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -24,6 +24,7 @@
 INSTANCE_FLAGS_IS_RVALUE   = 0x0004
 
 OVERLOAD_FLAGS_USE_FFI     = 0x0001
+OVERLOAD_FLAGS_CREATES     = 0x0002
 
 FUNCTION_IS_GLOBAL         = 0x0001
 FUNCTION_IS_STATIC         = 0x0001
@@ -229,8 +230,10 @@
         if self.converters is None:
             try:
                 self._setup(cppthis)
-            except Exception:
-                pass
+            except Exception as e:
+                if self.converters is None:
+                    raise oefmt(self.space.w_SystemError,
+                        "unable to initialize converters (%s)", str(e))
 
         # attempt to call directly through ffi chain
         if useffi and self._funcaddr:
@@ -458,6 +461,36 @@
 # need forwarding, which the normal instancemethod does not provide, hence this
 # derived class.
 class MethodWithProps(Method):
+    # set life management of result from the call
+    def fget_creates(self, space):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        return f.fget_creates(space)
+
+    @unwrap_spec(value=bool)
+    def fset_creates(self, space, value):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        f.fset_creates(space, value)
+
+    # set ownership policy of arguments (not yet implemented)
+    def fget_mempolicy(self, space):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        return f.fget_mempolicy(space)
+
+    @unwrap_spec(value=int)
+    def fset_mempolicy(self, space, value):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        f.fset_mempolicy(space, value)
+
+    # set to release the gil during call (not yet implemented)
+    def fget_release_gil(self, space):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        return f.fget_release_gil(space)
+
+    @unwrap_spec(value=bool)
+    def fset_release_gil(self, space, value):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        f.fset_release_gil(space, value)
+
     # allow user to determine ffi use rules per overload
     def fget_useffi(self, space):
         f = space.interp_w(W_CPPOverload, self.w_function)
@@ -473,19 +506,22 @@
     __doc__ = """cpp_instancemethod(function, instance, class)
 
 Create an instance method object.""",
-    __new__ = interp2app(MethodWithProps.descr_method__new__.im_func),
-    __call__ = interp2app(MethodWithProps.descr_method_call),
-    __get__ = interp2app(MethodWithProps.descr_method_get),
-    __func__ = interp_attrproperty_w('w_function', cls=MethodWithProps),
-    __self__ = interp_attrproperty_w('w_instance', cls=MethodWithProps),
+    __new__          = interp2app(MethodWithProps.descr_method__new__.im_func),
+    __call__         = interp2app(MethodWithProps.descr_method_call),
+    __get__          = interp2app(MethodWithProps.descr_method_get),
+    __func__         = interp_attrproperty_w('w_function', cls=MethodWithProps),
+    __self__         = interp_attrproperty_w('w_instance', cls=MethodWithProps),
     __getattribute__ = interp2app(MethodWithProps.descr_method_getattribute),
-    __eq__ = interp2app(MethodWithProps.descr_method_eq),
-    __ne__ = descr_generic_ne,
-    __hash__ = interp2app(MethodWithProps.descr_method_hash),
-    __repr__ = interp2app(MethodWithProps.descr_method_repr),
-    __reduce__ = interp2app(MethodWithProps.descr_method__reduce__),
-    __weakref__ = make_weakref_descr(MethodWithProps),
-    __useffi__ = GetSetProperty(MethodWithProps.fget_useffi, MethodWithProps.fset_useffi),
+    __eq__           = interp2app(MethodWithProps.descr_method_eq),
+    __ne__           = descr_generic_ne,
+    __hash__         = interp2app(MethodWithProps.descr_method_hash),
+    __repr__         = interp2app(MethodWithProps.descr_method_repr),
+    __reduce__       = interp2app(MethodWithProps.descr_method__reduce__),
+    __weakref__      = make_weakref_descr(MethodWithProps),
+    __creates__      = GetSetProperty(MethodWithProps.fget_creates,     MethodWithProps.fset_creates),
+    __mempolicy__    = GetSetProperty(MethodWithProps.fget_mempolicy,   MethodWithProps.fset_mempolicy),
+    __release_gil__  = GetSetProperty(MethodWithProps.fget_release_gil, MethodWithProps.fset_release_gil),
+    __useffi__       = GetSetProperty(MethodWithProps.fget_useffi,      MethodWithProps.fset_useffi),
     )
 MethodWithProps.typedef.acceptable_as_base_class = False
 
@@ -548,7 +584,12 @@
         for i in range(len(self.functions)):
             cppyyfunc = self.functions[i]
             try:
-                return cppyyfunc.call(cppthis, args_w, self.flags & OVERLOAD_FLAGS_USE_FFI)
+                w_result = cppyyfunc.call(cppthis, args_w, self.flags & OVERLOAD_FLAGS_USE_FFI)
+                if self.flags & OVERLOAD_FLAGS_CREATES:
+                    if isinstance(w_result, W_CPPInstance):
+                        cppinstance = self.space.interp_w(W_CPPInstance, w_result)
+                        cppinstance.fset_python_owns(self.space, self.space.w_True)
+                return w_result
             except Exception:
                 pass
 
@@ -561,6 +602,7 @@
         for i in range(len(self.functions)):
             cppyyfunc = self.functions[i]
             try:
+                # no need to set ownership on the return value, as none of the methods execute
                 return cppyyfunc.call(cppthis, args_w, self.flags & OVERLOAD_FLAGS_USE_FFI)
             except OperationError as e:
                 # special case if there's just one function, to prevent clogging the error message
@@ -599,6 +641,33 @@
                 return W_CPPOverload(self.space, self.scope, [f])
         raise oefmt(self.space.w_LookupError, "signature '%s' not found", signature)
 
+    # set life management of result from the call
+    def fget_creates(self, space):
+        return space.newbool(bool(self.flags & OVERLOAD_FLAGS_CREATES))
+
+    @unwrap_spec(value=bool)
+    def fset_creates(self, space, value):
+        if space.is_true(value):
+            self.flags |= OVERLOAD_FLAGS_CREATES
+        else:
+            self.flags &= ~OVERLOAD_FLAGS_CREATES
+
+    # set ownership policy of arguments (not yet implemented)
+    def fget_mempolicy(self, space):
+        return space.newint(0)
+
+    @unwrap_spec(value=int)
+    def fset_mempolicy(self, space, value):
+        pass
+
+    # set to release the gil during call (not yet implemented)
+    def fget_release_gil(self, space):
+        return space.newbool(True)
+
+    @unwrap_spec(value=bool)
+    def fset_release_gil(self, space, value):
+        pass
+
     # allow user to determine ffi use rules per overload
     def fget_useffi(self, space):
         return space.newbool(bool(self.flags & OVERLOAD_FLAGS_USE_FFI))
@@ -622,11 +691,14 @@
 
 W_CPPOverload.typedef = TypeDef(
     'CPPOverload',
-    __get__      = interp2app(W_CPPOverload.descr_get),
-    __call__     = interp2app(W_CPPOverload.call_args),
-    __useffi__   = GetSetProperty(W_CPPOverload.fget_useffi, W_CPPOverload.fset_useffi),
-    __overload__ = interp2app(W_CPPOverload.mp_overload),
-    __doc__      = GetSetProperty(W_CPPOverload.fget_doc)
+    __get__         = interp2app(W_CPPOverload.descr_get),
+    __call__        = interp2app(W_CPPOverload.call_args),
+    __creates__     = GetSetProperty(W_CPPOverload.fget_creates,     W_CPPOverload.fset_creates),
+    __mempolicy__   = GetSetProperty(W_CPPOverload.fget_mempolicy,   W_CPPOverload.fset_mempolicy),
+    __release_gil__ = GetSetProperty(W_CPPOverload.fget_release_gil, W_CPPOverload.fset_release_gil),
+    __useffi__      = GetSetProperty(W_CPPOverload.fget_useffi,      W_CPPOverload.fset_useffi),
+    __overload__    = interp2app(W_CPPOverload.mp_overload),
+    __doc__         = GetSetProperty(W_CPPOverload.fget_doc)
 )
 
 
@@ -655,11 +727,14 @@
 
 W_CPPStaticOverload.typedef = TypeDef(
     'CPPStaticOverload',
-    __get__      = interp2app(W_CPPStaticOverload.descr_get),
-    __call__     = interp2app(W_CPPStaticOverload.call_args),
-    __useffi__   = GetSetProperty(W_CPPStaticOverload.fget_useffi, W_CPPStaticOverload.fset_useffi),
-    __overload__ = interp2app(W_CPPStaticOverload.mp_overload),
-    __doc__      = GetSetProperty(W_CPPStaticOverload.fget_doc)
+    __get__         = interp2app(W_CPPStaticOverload.descr_get),
+    __call__        = interp2app(W_CPPStaticOverload.call_args),
+    __creates__     = GetSetProperty(W_CPPStaticOverload.fget_creates,     W_CPPStaticOverload.fset_creates),
+    __mempolicy__   = GetSetProperty(W_CPPStaticOverload.fget_mempolicy,   W_CPPStaticOverload.fset_mempolicy),
+    __release_gil__ = GetSetProperty(W_CPPStaticOverload.fget_release_gil, W_CPPStaticOverload.fset_release_gil),
+    __useffi__      = GetSetProperty(W_CPPStaticOverload.fget_useffi,      W_CPPStaticOverload.fset_useffi),
+    __overload__    = interp2app(W_CPPStaticOverload.mp_overload),
+    __doc__         = GetSetProperty(W_CPPStaticOverload.fget_doc)
 )
 
 
@@ -712,6 +787,7 @@
 class TemplateOverloadMixin(object):
     """Mixin to instantiate templated methods/functions."""
 
+    _attrs_ = ['tmpl_args_w']
     _mixin_ = True
 
     def construct_template_args(self, w_tpArgs, args_w = None):
@@ -764,23 +840,37 @@
         return cppol
 
     def instantiate_and_call(self, name, args_w):
-        # try to match with run-time instantiations
-        for cppol in self.master.overloads.values():
-            try:
-                if not self.space.is_w(self.w_this, self.space.w_None):
-                    return self.space.call_obj_args(cppol, self.w_this, Arguments(self.space, args_w))
-                return self.space.call_args(cppol, Arguments(self.space, args_w))
-            except Exception:
-                pass    # completely ignore for now; have to see whether errors become confusing
+        method = None
+        try:
+            # existing cached instantiations
+            if name[-1] == '>':   # only accept full templated name, to ensure explicit
+                method = self.master.overloads[name]
+            else:
+            # try to match with run-time instantiations
+                # TODO: logically, this could be used, but in practice, it's proving too
+                #  greedy ... maybe as a last resort?
+                #for cppol in self.master.overloads.values():
+                #    try:
+                #        if not self.space.is_w(self.w_this, self.space.w_None):
+                #            return self.space.call_obj_args(cppol, self.w_this, Arguments(self.space, args_w))
+                #        return self.space.call_args(cppol, Arguments(self.space, args_w))
+                #    except Exception:
+                #        pass    # completely ignore for now; have to see whether errors become confusing
+                raise TypeError("pre-existing overloads failed")
+        except (KeyError, TypeError):
+            # if not known, try to deduce from argument types
+            w_types = self.space.newtuple([self.space.type(obj_w) for obj_w in args_w])
+            proto = self.construct_template_args(w_types, args_w)
+            method = self.find_method_template(name, proto)
 
-        # if all failed, then try to deduce from argument types
-        w_types = self.space.newtuple([self.space.type(obj_w) for obj_w in args_w])
-        proto = self.construct_template_args(w_types, args_w)
-        method = self.find_method_template(name, proto)
-
-        # only cache result if the name retains the full template
-        fullname = capi.c_method_full_name(self.space, method.functions[0].cppmethod)
-        if 0 <= fullname.rfind('>'):
+            # only cache result if the name retains the full template
+            # TODO: the problem is in part that c_method_full_name returns incorrect names,
+            # e.g. when default template arguments are involved, so for now use 'name' if it
+            # has the full templated name
+            if name[-1] == '>':
+                fullname = name
+            else:
+                fullname = capi.c_method_full_name(self.space, method.functions[0].cppmethod)
             try:
                 existing = self.master.overloads[fullname]
                 allf = existing.functions + method.functions
@@ -792,9 +882,10 @@
             except KeyError:
                 self.master.overloads[fullname] = method
 
-        if not self.space.is_w(self.w_this, self.space.w_None):
-            return self.space.call_obj_args(method, self.w_this, Arguments(self.space, args_w))
-        return self.space.call_args(method, Arguments(self.space, args_w))
+        if method is not None:
+            if not self.space.is_w(self.w_this, self.space.w_None):
+                return self.space.call_obj_args(method, self.w_this, Arguments(self.space, args_w))
+            return self.space.call_args(method, Arguments(self.space, args_w))
 
     def getitem_impl(self, name, args_w):
         space = self.space
@@ -808,14 +899,9 @@
         fullname = name+'<'+tmpl_args+'>'
         try:
             method = self.master.overloads[fullname]
-        except KeyError:
-            method = self.find_method_template(fullname)
-            # cache result (name is always full templated name)
-            self.master.overloads[fullname] = method
-            # also cache on "official" name (may include default template arguments)
-            c_fullname = capi.c_method_full_name(self.space, method.functions[0].cppmethod)
-            if c_fullname != fullname:
-                self.master.overloads[c_fullname] = method
+        except KeyError as e:
+            # defer instantiation until arguments are known
+            return self.clone(tmpl_args)
 
         return method.descr_get(self.w_this, None)
 
@@ -824,21 +910,29 @@
     """App-level dispatcher to allow both lookup/instantiation of templated methods and
     dispatch among overloads between templated and non-templated method."""
 
-    _attrs_ = ['name', 'overloads', 'master', 'w_this']
-    _immutable_fields_ = ['name']
+    _attrs_ = ['name', 'tmpl_args', 'overloads', 'master', 'w_this']
+    _immutable_fields_ = ['name', 'tmpl_args']
 
-    def __init__(self, space, name, decl_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
+    def __init__(self, space, name, tmpl_args, decl_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
          W_CPPOverload.__init__(self, space, decl_scope, functions, flags)
          self.name = name
+         self.tmpl_args = tmpl_args
          self.overloads = {}
          self.master = self
          self.w_this = space.w_None
 
+    def clone(self, tmpl_args):
+        other = W_CPPTemplateOverload(self.space, self.name, tmpl_args, self.scope, self.functions, self.flags)
+        other.overloads = self.overloads
+        other.master = self.master
+        other.w_this = self.w_this
+        return other
+
     def descr_get(self, w_cppinstance, w_cls=None):
         # TODO: don't return copy, but bind in an external object (like W_CPPOverload)
         if self.space.is_w(w_cppinstance, self.space.w_None):
             return self  # unbound, so no new instance needed
-        cppol = W_CPPTemplateOverload(self.space, self.name, self.scope, self.functions, self.flags)
+        cppol = W_CPPTemplateOverload(self.space, self.name, self.tmpl_args, self.scope, self.functions, self.flags)
         cppol.w_this = w_cppinstance
         cppol.master = self.master
         return cppol     # bound
@@ -848,13 +942,18 @@
         # direct call: either pick non-templated overload or attempt to deduce
         # the template instantiation from the argument types
 
-        # try existing overloads or compile-time instantiations
+        # do explicit lookup with tmpl_args if given
         try:
-            return W_CPPOverload.call_args(self, args_w)
+            fullname = self.name
+            if self.tmpl_args is not None:
+                fullname = fullname+'<'+self.tmpl_args+'>'
+            return self.instantiate_and_call(fullname, args_w)
         except Exception:
             pass
 
-        return self.instantiate_and_call(self.name, args_w)
+        # otherwise, try existing overloads or compile-time instantiations
+        # TODO: consolidate errors
+        return W_CPPOverload.call_args(self, [self.w_this]+args_w)
 
     @unwrap_spec(args_w='args_w')
     def getitem(self, args_w):
@@ -868,33 +967,44 @@
 
 W_CPPTemplateOverload.typedef = TypeDef(
     'CPPTemplateOverload',
-    __get__      = interp2app(W_CPPTemplateOverload.descr_get),
-    __getitem__  = interp2app(W_CPPTemplateOverload.getitem),
-    __call__     = interp2app(W_CPPTemplateOverload.call_args),
-    __useffi__   = GetSetProperty(W_CPPTemplateOverload.fget_useffi, W_CPPTemplateOverload.fset_useffi),
-    __doc__      = GetSetProperty(W_CPPTemplateOverload.fget_doc)
+    __get__         = interp2app(W_CPPTemplateOverload.descr_get),
+    __getitem__     = interp2app(W_CPPTemplateOverload.getitem),
+    __call__        = interp2app(W_CPPTemplateOverload.call_args),
+    __creates__     = GetSetProperty(W_CPPTemplateOverload.fget_creates,     W_CPPTemplateOverload.fset_creates),
+    __mempolicy__   = GetSetProperty(W_CPPTemplateOverload.fget_mempolicy,   W_CPPTemplateOverload.fset_mempolicy),
+    __release_gil__ = GetSetProperty(W_CPPTemplateOverload.fget_release_gil, W_CPPTemplateOverload.fset_release_gil),
+    __useffi__      = GetSetProperty(W_CPPTemplateOverload.fget_useffi,      W_CPPTemplateOverload.fset_useffi),
+    __doc__         = GetSetProperty(W_CPPTemplateOverload.fget_doc)
 )
 
 class W_CPPTemplateStaticOverload(W_CPPStaticOverload, TemplateOverloadMixin):
     """App-level dispatcher to allow both lookup/instantiation of templated methods and
     dispatch among overloads between templated and non-templated method."""
 
-    _attrs_ = ['name', 'overloads', 'master', 'w_this']
-    _immutable_fields_ = ['name']
+    _attrs_ = ['name', 'tmpl_args', 'overloads', 'master', 'w_this']
+    _immutable_fields_ = ['name', 'tmpl_args']
 
-    def __init__(self, space, name, decl_scope, funcs, flags = OVERLOAD_FLAGS_USE_FFI):
+    def __init__(self, space, name, tmpl_args, decl_scope, funcs, flags = OVERLOAD_FLAGS_USE_FFI):
          W_CPPStaticOverload.__init__(self, space, decl_scope, funcs, flags)
          self.name = name
+         self.tmpl_args = tmpl_args
          self.overloads = {}
          self.master = self
          self.w_this = space.w_None
 
+    def clone(self, tmpl_args):
+        other = W_CPPTemplateStaticOverload(self.space, self.name, tmpl_args, self.scope, self.functions, self.flags)
+        other.overloads = self.overloads
+        other.master = self.master
+        other.w_this = self.w_this
+        return other
+
     def descr_get(self, w_cppinstance, w_cls=None):
         # TODO: don't return copy, but bind in an external object (like W_CPPOverload)
         if isinstance(w_cppinstance, W_CPPInstance):
             cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance)
             if cppinstance.clsdecl.handle != self.scope.handle:
-                cppol = W_CPPTemplateStaticOverload(self.space, self.name, self.scope, self.functions, self.flags)
+                cppol = W_CPPTemplateStaticOverload(self.space, self.name, self.tmpl_args, self.scope, self.functions, self.flags)
                 cppol.w_this = w_cppinstance
                 cppol.master = self.master
                 return cppol       # bound
@@ -904,15 +1014,20 @@
     def call_args(self, args_w):
         # direct call: either pick non-templated overload or attempt to deduce
         # the template instantiation from the argument types
+        # TODO: refactor with W_CPPTemplateOverload
 
-        # try existing overloads or compile-time instantiations
+        # do explicit lookup with tmpl_args if given
         try:
-            return W_CPPStaticOverload.call_args(self, args_w)
+            fullname = self.name
+            if self.tmpl_args is not None:
+                fullname = fullname+'<'+self.tmpl_args+'>'
+            return self.instantiate_and_call(fullname, args_w)
         except Exception:
             pass
 
-        # try new instantiation
-        return self.instantiate_and_call(self.name, args_w)
+        # otherwise, try existing overloads or compile-time instantiations
+        # TODO: consolidate errors
+        return W_CPPStaticOverload.call_args(self, args_w)
 
     @unwrap_spec(args_w='args_w')
     def getitem(self, args_w):
@@ -926,11 +1041,18 @@
 
 W_CPPTemplateStaticOverload.typedef = TypeDef(
     'CPPTemplateStaticOverload',
-    __get__      = interp2app(W_CPPTemplateStaticOverload.descr_get),
-    __getitem__  = interp2app(W_CPPTemplateStaticOverload.getitem),
-    __call__     = interp2app(W_CPPTemplateStaticOverload.call_args),
-    __useffi__   = GetSetProperty(W_CPPTemplateStaticOverload.fget_useffi, W_CPPTemplateStaticOverload.fset_useffi),
-    __doc__      = GetSetProperty(W_CPPTemplateStaticOverload.fget_doc)
+    __get__         = interp2app(W_CPPTemplateStaticOverload.descr_get),
+    __getitem__     = interp2app(W_CPPTemplateStaticOverload.getitem),
+    __call__        = interp2app(W_CPPTemplateStaticOverload.call_args),
+    __creates__     = GetSetProperty(W_CPPTemplateStaticOverload.fget_creates,
+                                     W_CPPTemplateStaticOverload.fset_creates),
+    __mempolicy__   = GetSetProperty(W_CPPTemplateStaticOverload.fget_mempolicy,
+                                     W_CPPTemplateStaticOverload.fset_mempolicy),
+    __release_gil__ = GetSetProperty(W_CPPTemplateStaticOverload.fget_release_gil,
+                                     W_CPPTemplateStaticOverload.fset_release_gil),
+    __useffi__      = GetSetProperty(W_CPPTemplateStaticOverload.fget_useffi,
+                                     W_CPPTemplateStaticOverload.fset_useffi),
+    __doc__         = GetSetProperty(W_CPPTemplateStaticOverload.fget_doc)
 )
 
 
@@ -950,11 +1072,11 @@
     _attrs_ = ['space', 'scope', 'converter', 'offset']
     _immutable_fields = ['scope', 'converter', 'offset']
 
-    def __init__(self, space, decl_scope, type_name, offset):
+    def __init__(self, space, decl_scope, type_name, dimensions, offset):
         self.space = space
         self.scope = decl_scope
-        self.converter = converter.get_converter(self.space, type_name, '')
-        self.offset = offset
+        self.converter = converter.get_converter(self.space, type_name, dimensions)
+        self.offset = rffi.cast(rffi.LONG, offset)
 
     def _get_offset(self, cppinstance):
         if cppinstance:
@@ -971,7 +1093,7 @@
             raise oefmt(self.space.w_AttributeError,
                         "attribute access requires an instance")
         offset = self._get_offset(cppinstance)
-        return self.converter.from_memory(self.space, w_cppinstance, w_pycppclass, offset)
+        return self.converter.from_memory(self.space, w_cppinstance, offset)
 
     def set(self, w_cppinstance, w_value):
         cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=True)
@@ -1008,7 +1130,7 @@
         return self.offset
 
     def get(self, w_cppinstance, w_pycppclass):
-        return self.converter.from_memory(self.space, self.space.w_None, w_pycppclass, self.offset)
+        return self.converter.from_memory(self.space, self.space.w_None, self.offset)
 
     def set(self, w_cppinstance, w_value):
         self.converter.to_memory(self.space, self.space.w_None, w_value, self.offset)
@@ -1098,6 +1220,21 @@
         self.datamembers[name] = new_dm
         return new_dm
 
+    @unwrap_spec(name='text')
+    def has_enum(self, name):
+        if capi.c_is_enum(self.space, self.name+'::'+name):
+            return self.space.w_True
+        return self.space.w_False
+
+    def _encode_dm_dimensions(self, idata):
+        # encode dimensions (TODO: this is ugly, but here's where the info is)
+        dims = []
+        sz = capi.c_get_dimension_size(self.space, self, idata, len(dims))
+        while 0 < sz:
+            dims.append(str(sz))
+            sz = capi.c_get_dimension_size(self.space, self, idata, len(dims))
+        return ':'.join(dims)
+
     @unwrap_spec(name='text', signature='text')
     def scope__dispatch__(self, name, signature):
         overload = self.get_overload(name)
@@ -1137,13 +1274,16 @@
 
     def _make_datamember(self, dm_name, dm_idx):
         type_name = capi.c_datamember_type(self.space, self, dm_idx)
+        if capi.c_is_enum_data(self.space, self, dm_idx):
+            type_name = capi.c_resolve_enum(self.space, type_name)
         offset = capi.c_datamember_offset(self.space, self, dm_idx)
         if offset == -1:
             raise self.missing_attribute_error(dm_name)
+        dims = self._encode_dm_dimensions(dm_idx)
         if capi.c_is_const_data(self.space, self, dm_idx):
-            datamember = W_CPPConstStaticData(self.space, self, type_name, offset)
+            datamember = W_CPPConstStaticData(self.space, self, type_name, dims, offset)
         else:
-            datamember = W_CPPStaticData(self.space, self, type_name, offset)
+            datamember = W_CPPStaticData(self.space, self, type_name, dims, offset)
         self.datamembers[dm_name] = datamember
         return datamember
 
@@ -1158,10 +1298,10 @@
                 if capi.c_method_is_template(self.space, self, idx):
                     templated = True
             if templated:
-                return W_CPPTemplateStaticOverload(self.space, meth_name, self, cppfunctions[:])
+                return W_CPPTemplateStaticOverload(self.space, meth_name, None, self, cppfunctions[:])
             return W_CPPStaticOverload(self.space, self, cppfunctions[:])
         elif capi.c_exists_method_template(self.space, self, meth_name):
-            return W_CPPTemplateStaticOverload(self.space, meth_name, self, [])
+            return W_CPPTemplateStaticOverload(self.space, meth_name, None, self, [])
         raise self.missing_attribute_error(meth_name)
 
     def find_datamember(self, dm_name):
@@ -1193,6 +1333,7 @@
     get_datamember_names = interp2app(W_CPPNamespaceDecl.get_datamember_names),
     get_datamember = interp2app(W_CPPNamespaceDecl.get_datamember),
     is_namespace = interp2app(W_CPPNamespaceDecl.is_namespace),
+    has_enum = interp2app(W_CPPNamespaceDecl.has_enum),
     __cppname__ = interp_attrproperty('name', W_CPPNamespaceDecl, wrapfn="newtext"),
     __dispatch__ = interp2app(W_CPPNamespaceDecl.scope__dispatch__),
     __dir__ = interp2app(W_CPPNamespaceDecl.ns__dir__),
@@ -1253,12 +1394,12 @@
             elif ftype & FUNCTION_IS_STATIC:
                 if ftype & FUNCTION_IS_TEMPLATE:
                     cppname = capi.c_method_name(self.space, methods[0].cppmethod)
-                    overload = W_CPPTemplateStaticOverload(self.space, cppname, self, methods[:])
+                    overload = W_CPPTemplateStaticOverload(self.space, cppname, None, self, methods[:])
                 else:
                     overload = W_CPPStaticOverload(self.space, self, methods[:])
             elif ftype & FUNCTION_IS_TEMPLATE:
                 cppname = capi.c_method_name(self.space, methods[0].cppmethod)
-                overload = W_CPPTemplateOverload(self.space, cppname, self, methods[:])
+                overload = W_CPPTemplateOverload(self.space, cppname, None, self, methods[:])
             else:
                 overload = W_CPPOverload(self.space, self, methods[:])
             self.overloads[pyname] = overload
@@ -1298,14 +1439,15 @@
                 continue      # dictionary problem; raises AttributeError on use
             is_static = bool(capi.c_is_staticdata(self.space, self, i))
             is_const  = bool(capi.c_is_const_data(self.space, self, i))
+            dims = self._encode_dm_dimensions(i)
             if is_static and is_const:
-                datamember = W_CPPConstStaticData(self.space, self, type_name, offset)
+                datamember = W_CPPConstStaticData(self.space, self, type_name, dims, offset)
             elif is_static:
-                datamember = W_CPPStaticData(self.space, self, type_name, offset)
+                datamember = W_CPPStaticData(self.space, self, type_name, dims, offset)
             elif is_const:
-                datamember = W_CPPConstDataMember(self.space, self, type_name, offset)
+                datamember = W_CPPConstDataMember(self.space, self, type_name, dims, offset)
             else:
-                datamember = W_CPPDataMember(self.space, self, type_name, offset)
+                datamember = W_CPPDataMember(self.space, self, type_name, dims, offset)
             self.datamembers[datamember_name] = datamember
 
     def find_overload(self, meth_name):
@@ -1348,6 +1490,7 @@
     get_datamember_names = interp2app(W_CPPClassDecl.get_datamember_names),
     get_datamember = interp2app(W_CPPClassDecl.get_datamember),
     is_namespace = interp2app(W_CPPClassDecl.is_namespace),
+    has_enum = interp2app(W_CPPClassDecl.has_enum),
     __cppname__ = interp_attrproperty('name', W_CPPClassDecl, wrapfn="newtext"),
     __dispatch__ = interp2app(W_CPPClassDecl.scope__dispatch__)
 )
diff --git a/pypy/module/_cppyy/lowlevelviews.py b/pypy/module/_cppyy/lowlevelviews.py
--- a/pypy/module/_cppyy/lowlevelviews.py
+++ b/pypy/module/_cppyy/lowlevelviews.py
@@ -3,10 +3,17 @@
 # a few more methods allowing such information to be set. Afterwards, it is
 # simple to pass these views on to e.g. numpy (w/o the need to copy).
 
-from pypy.interpreter.error import oefmt
+from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty_w
+from pypy.interpreter.baseobjspace import W_Root
+
+from rpython.rtyper.lltypesystem import rffi
+from rpython.rlib.rarithmetic import intmask
+
 from pypy.module._rawffi.array import W_ArrayInstance
+from pypy.module._rawffi.interp_rawffi import segfault_exception
+from pypy.module._cppyy import capi
 
 
 class W_LowLevelView(W_ArrayInstance):
@@ -50,5 +57,45 @@
     itemaddress = interp2app(W_LowLevelView.descr_itemaddress),
     reshape     = interp2app(W_LowLevelView.reshape),
 )
-W_ArrayInstance.typedef.acceptable_as_base_class = False
+W_LowLevelView.typedef.acceptable_as_base_class = False
 
+
+class W_ArrayOfInstances(W_Root):
+    _attrs_ = ['converter', 'baseaddress', 'clssize', 'length']
+    _immutable_fields_ = ['converter', 'baseaddress', 'clssize']
+
+    def __init__(self, space, clsdecl, address, length, dimensions):
+        from pypy.module._cppyy import converter
+        name = clsdecl.name
+        self.clssize = int(intmask(capi.c_size_of_klass(space, clsdecl)))
+        if dimensions:
+            name = name + '[' + dimensions[0] + ']'
+            for num in dimensions:
+                self.clssize *= int(num)
+        dimensions = ':'.join(dimensions)
+        self.converter   = converter.get_converter(space, name, dimensions)
+        self.baseaddress = address
+        self.length      = length
+
+    @unwrap_spec(idx=int)
+    def getitem(self, space, idx):
+        if not self.baseaddress:
+            raise segfault_exception(space, "accessing elements of freed array")
+        if idx >= self.length or idx < 0:
+            raise OperationError(space.w_IndexError, space.w_None)
+        itemaddress = rffi.cast(rffi.LONG, self.baseaddress)+idx*self.clssize
+        return self.converter.from_memory(space, space.w_None, itemaddress)
+
+    def getlength(self, space):
+        return space.newint(self.length)
+
+    def setlength(self, space, w_length):
+        self.length = space.int_w(w_length)
+
+W_ArrayOfInstances.typedef = TypeDef(
+    'ArrayOfInstances',
+    __getitem__ = interp2app(W_ArrayOfInstances.getitem),
+    __len__     = interp2app(W_ArrayOfInstances.getlength),
+    size        = GetSetProperty(W_ArrayOfInstances.getlength, W_ArrayOfInstances.setlength),
+)
+W_ArrayOfInstances.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/_cppyy/pythonify.py b/pypy/module/_cppyy/pythonify.py
--- a/pypy/module/_cppyy/pythonify.py
+++ b/pypy/module/_cppyy/pythonify.py
@@ -29,7 +29,14 @@
 
 class CPPNamespaceMeta(CPPScopeMeta):
     def __dir__(self):
-        return self.__cppdecl__.__dir__()
+        # For Py3: can actually call base class __dir__ (lives in type)
+        values = set(self.__dict__.keys())
+        values.update(object.__dict__.keys())
+        values.update(type(self).__dict__.keys())
+
+        # add C++ entities
+        values.update(self.__cppdecl__.__dir__())
+        return list(values)
 
 class CPPClassMeta(CPPScopeMeta):
     pass
@@ -310,7 +317,7 @@
     if not cppitem:
         try:
             cppitem = scope.__cppdecl__.get_overload(name)
-            setattr(scope.__class__, name, cppitem)
+            setattr(scope, name, cppitem)
             pycppitem = getattr(scope, name)      # binds function as needed
         except AttributeError:
             pass
@@ -326,6 +333,11 @@
         except AttributeError:
             pass
 
+    # enum type
+    if not cppitem:
+        if scope.__cppdecl__.has_enum(name):
+            pycppitem = int
+
     if pycppitem is not None:      # pycppitem could be a bound C++ NULL, so check explicitly for Py_None
         return pycppitem
 
@@ -354,18 +366,19 @@
     return '', name
 
 # pythonization by decoration (move to their own file?)
-def python_style_getitem(self, idx):
+def python_style_getitem(self, _idx):
     # python-style indexing: check for size and allow indexing from the back
-    try:
-        sz = len(self)
+    sz = len(self)
+    idx = _idx
+    if isinstance(idx, int):
         if idx < 0: idx = sz + idx
-        if idx < sz:
+        if 0 <= idx < sz:
             return self._getitem__unchecked(idx)
-        raise IndexError(
-            'index out of range: %d requested for %s of size %d' % (idx, str(self), sz))
-    except TypeError:
-        pass
-    return self._getitem__unchecked(idx)
+        else:
+            raise IndexError(
+                'index out of range: %s requested for %s of size %d' % (str(idx), str(self), sz))
+    # may fail for the same reasons as above, but will now give proper error message
+    return self._getitem__unchecked(_idx)
 
 def python_style_sliceable_getitem(self, slice_or_idx):
     if type(slice_or_idx) == slice:
@@ -373,8 +386,7 @@
         nseq += [python_style_getitem(self, i) \
                     for i in range(*slice_or_idx.indices(len(self)))]
         return nseq
-    else:
-        return python_style_getitem(self, slice_or_idx)
+    return python_style_getitem(self, slice_or_idx)
 
 def _pythonize(pyclass, name):
     # general note: use 'in pyclass.__dict__' rather than 'hasattr' to prevent
@@ -423,22 +435,25 @@
     # map begin()/end() protocol to iter protocol on STL(-like) classes, but
     # not on vector, which is pythonized in the capi (interp-level; there is
     # also the fallback on the indexed __getitem__, but that is slower)
-# TODO:    if not (0 <= name.find('vector') <= 5):
-    if ('begin' in pyclass.__dict__ and 'end' in pyclass.__dict__):
-        if _cppyy._scope_byname(name+'::iterator') or \
-                _cppyy._scope_byname(name+'::const_iterator'):
-            def __iter__(self):
-                i = self.begin()
-                while i != self.end():
-                    yield i.__deref__()
-                    i.__preinc__()
-                i.__destruct__()
-                raise StopIteration
-            pyclass.__iter__ = __iter__
-        # else: rely on numbered iteration
+    add_checked_item = False
+    if name.find('std::vector', 0, 11) != 0:
+        if ('begin' in pyclass.__dict__ and 'end' in pyclass.__dict__):
+            if _cppyy._scope_byname(name+'::iterator') or \
+                    _cppyy._scope_byname(name+'::const_iterator'):
+                def __iter__(self):
+                    i = self.begin()
+                    while i != self.end():
+                        yield i.__deref__()
+                        i.__preinc__()
+                    i.__destruct__()
+                    raise StopIteration
+                pyclass.__iter__ = __iter__
+            else:
+                # rely on numbered iteration
+                add_checked_item = True
 
     # add python collection based initializer
-    if 0 <= name.find('vector') <= 5:
+    if name.find('std::vector', 0, 11) == 0:
         pyclass.__real_init__ = pyclass.__init__
         def vector_init(self, *args):
             if len(args) == 1 and isinstance(args[0], (tuple, list)):
@@ -452,20 +467,25 @@
         pyclass.__init__ = vector_init
 
         # size-up the return of data()
-        pyclass.__real_data = pyclass.data
-        def data_with_len(self):
-            arr = self.__real_data()
-            arr.reshape((len(self),))
-            return arr
-        pyclass.data = data_with_len
+        if hasattr(pyclass, 'data'):   # not the case for e.g. vector<bool>
+            pyclass.__real_data = pyclass.data
+            def data_with_len(self):
+                arr = self.__real_data()
+                arr.reshape((len(self),))
+                return arr
+            pyclass.data = data_with_len
 
-    # combine __getitem__ and __len__ to make a pythonized __getitem__
-    if '__getitem__' in pyclass.__dict__ and '__len__' in pyclass.__dict__:
-        pyclass._getitem__unchecked = pyclass.__getitem__
-        if '__setitem__' in pyclass.__dict__ and '__iadd__' in pyclass.__dict__:
-            pyclass.__getitem__ = python_style_sliceable_getitem
-        else:
-            pyclass.__getitem__ = python_style_getitem
+    # TODO: must be a simpler way to check (or at least hook these to a namespace
+    # std specific pythonizor)
+    if add_checked_item or name.find('std::vector', 0, 11) == 0 or \
+            name.find('std::array', 0, 11) == 0 or name.find('std::deque', 0, 10) == 0:
+        # combine __getitem__ and __len__ to make a pythonized __getitem__
+        if '__getitem__' in pyclass.__dict__ and '__len__' in pyclass.__dict__:
+            pyclass._getitem__unchecked = pyclass.__getitem__
+            if '__setitem__' in pyclass.__dict__ and '__iadd__' in pyclass.__dict__:
+                pyclass.__getitem__ = python_style_sliceable_getitem
+            else:
+                pyclass.__getitem__ = python_style_getitem
 
     # string comparisons
     if name == _cppyy._std_string_name():
diff --git a/pypy/module/_cppyy/src/dummy_backend.cxx b/pypy/module/_cppyy/src/dummy_backend.cxx
--- a/pypy/module/_cppyy/src/dummy_backend.cxx
+++ b/pypy/module/_cppyy/src/dummy_backend.cxx
@@ -62,6 +62,7 @@
         m_name(name), m_argtypes(argtypes), m_returntype(returntype), m_type(mtype) {}
     std::string m_name;
     std::vector<std::string> m_argtypes;
+    std::vector<std::string> m_argdefaults;
     std::string m_returntype;
     EMethodType m_type;
 };
@@ -376,6 +377,13 @@
         PUBLIC_CPPYY_STATIC_DATA(enum,    CppyyTestData::EWhat);
         PUBLIC_CPPYY_STATIC_DATA(voidp,   void*);
 
+      // default tester for long double
+        argtypes.clear();
+        argtypes.push_back("long double");
+        methods.push_back(new Cppyy_PseudoMethodInfo("get_ldouble_def", argtypes, "long double"));
+        methods.back()->m_argdefaults.push_back("aap_t(1)");
+        s_methods["CppyyTestData::get_ldouble_def"] = methods.back();
+
       // pretend enum values
         data.push_back(Cppyy_PseudoDatambrInfo(
             "kNothing", "CppyyTestData::EWhat", (ptrdiff_t)&Pseudo_kNothing, true));
@@ -388,6 +396,16 @@
         s_scopes[(cppyy_scope_t)s_scope_id] = info;
         } // -- class CppyyTest_data
 
+        //====================================================================
+
+        { // namespace pyzables --
+        s_handles["pyzables"] = (cppyy_scope_t)++s_scope_id;
+        s_scopes[(cppyy_scope_t)s_scope_id] = Cppyy_PseudoClassInfo{};
+        s_handles["pyzables::SomeDummy1"] = (cppyy_scope_t)++s_scope_id;
+        s_scopes[(cppyy_scope_t)s_scope_id] = Cppyy_PseudoClassInfo{};
+        s_handles["pyzables::SomeDummy2"] = (cppyy_scope_t)++s_scope_id;
+        s_scopes[(cppyy_scope_t)s_scope_id] = Cppyy_PseudoClassInfo{};
+        } // -- namespace pyzables
     }
 } _init;
 
@@ -406,6 +424,8 @@
 char* cppyy_resolve_name(const char* cppitem_name) {
     if (cppyy_is_enum(cppitem_name))
         return cppstring_to_cstring("internal_enum_type_t");
+    else if (strcmp(cppitem_name, "aap_t") == 0)
+        return cppstring_to_cstring("long double");
     return cppstring_to_cstring(cppitem_name);
 }
 
@@ -510,6 +530,12 @@
     } else if (idx == s_methods["CppyyTestData::set_double_cr"]) {
         assert(self && nargs == 1);
         ((dummy::CppyyTestData*)self)->set_double_cr(*(double*)&((CPPYY_G__value*)args)[0]);
+    } else if (idx == s_methods["CppyyTestData::set_ldouble"]) {
+        assert(self && nargs == 1);
+        ((dummy::CppyyTestData*)self)->set_ldouble(((CPPYY_G__value*)args)[0].obj.ld);
+    } else if (idx == s_methods["CppyyTestData::set_ldouble_cr"]) {
+        assert(self && nargs == 1);
+        ((dummy::CppyyTestData*)self)->set_ldouble_cr(*(long double*)&((CPPYY_G__value*)args)[0]);
     } else {
         assert(!"method unknown in cppyy_call_v");
     }
@@ -784,6 +810,26 @@
     return result;
 }
 
+double cppyy_call_nld(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
+    double result = 0.;
+    Cppyy_PseudoMethodInfo* idx = (Cppyy_PseudoMethodInfo*)method;
+    if (idx == s_methods["CppyyTestData::get_ldouble_def"]) {
+        if (nargs == 1)
+            result = (double)((dummy::CppyyTestData*)self)->get_ldouble_def(
+                ((CPPYY_G__value*)args)[0].obj.ld);
+        else {
+            assert(self && nargs == 0);
+            result = (double)((dummy::CppyyTestData*)self)->get_ldouble_def();
+        }
+    } else if (idx == s_methods["CppyyTestData::get_ldouble"]) {
+        assert(self && nargs == 0);
+        result = (double)((dummy::CppyyTestData*)self)->get_ldouble();
+    } else {
+        assert(!"method unknown in cppyy_call_nld");
+    }
+    return result;
+}
+
 #define DISPATCH_CALL_R_GET(tpname)                                           \
     else if (idx == s_methods["CppyyTestData::get_"#tpname"_r"]) {            \
         assert(self && nargs == 0);                                           \
@@ -879,7 +925,9 @@
 
 
 /* scope reflection information ------------------------------------------- */
-int cppyy_is_namespace(cppyy_scope_t /* handle */) {
+int cppyy_is_namespace(cppyy_scope_t handle) {
+    if (handle == s_handles["pyzables"])
+        return 1;
     return 0;
 }
 
@@ -954,14 +1002,16 @@
 }
 
 int cppyy_method_req_args(cppyy_method_t method) {
-    return cppyy_method_num_args(method);
+    return cppyy_method_num_args(method)-((Cppyy_PseudoMethodInfo*)method)->m_argdefaults.size();
 }
 
 char* cppyy_method_arg_type(cppyy_method_t method, int idx) {
     return cppstring_to_cstring(((Cppyy_PseudoMethodInfo*)method)->m_argtypes[idx]);
 }
 
-char* cppyy_method_arg_default(cppyy_method_t, int /* arg_index */) {
+char* cppyy_method_arg_default(cppyy_method_t method, int idx) {
+    if (idx < (int)((Cppyy_PseudoMethodInfo*)method)->m_argdefaults.size())
+        return cppstring_to_cstring(((Cppyy_PseudoMethodInfo*)method)->m_argdefaults[idx]);
     return cppstring_to_cstring("");
 }
 
@@ -1048,6 +1098,11 @@
     return 0;
 }
 
+int cppyy_get_dimension_size(
+        cppyy_scope_t /* scope */, cppyy_index_t /* idata */, int /* dimension */) {
+    return -1; // no dimensions
+}
+
 
 /* misc helpers ----------------------------------------------------------- */
 #if defined(_MSC_VER)
@@ -1077,13 +1132,30 @@
 }
 
 cppyy_object_t cppyy_charp2stdstring(const char* str, size_t sz) {
-    void* arena = new char[sz];
-    new (arena) std::string(str, sz);
-    return (cppyy_object_t)arena;
+    return (cppyy_object_t)new std::string(str, sz);
+}
+
+const char* cppyy_stdstring2charp(cppyy_object_t ptr, size_t* lsz) {
+    *lsz = ((std::string*)ptr)->size();
+    return ((std::string*)ptr)->data();
 }
 
 cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr) {
-    void* arena = new char[sizeof(std::string)];
-    new (arena) std::string(*(std::string*)ptr);
-    return (cppyy_object_t)arena;
+    return (cppyy_object_t)new std::string(*(std::string*)ptr);
 }
+
+double cppyy_longdouble2double(void* p) {
+    return (double)*(long double*)p;
+}
+
+void cppyy_double2longdouble(double d, void* p) {
+    *(long double*)p = d;
+}
+
+int cppyy_vectorbool_getitem(cppyy_object_t ptr, int idx) {
+    return (int)(*(std::vector<bool>*)ptr)[idx];
+}
+
+void cppyy_vectorbool_setitem(cppyy_object_t ptr, int idx, int value) {
+    (*(std::vector<bool>*)ptr)[idx] = (bool)value;
+}
diff --git a/pypy/module/_cppyy/test/conftest.py b/pypy/module/_cppyy/test/conftest.py
--- a/pypy/module/_cppyy/test/conftest.py
+++ b/pypy/module/_cppyy/test/conftest.py
@@ -8,20 +8,25 @@
             # run only tests that are covered by the dummy backend and tests
             # that do not rely on reflex
             import os
+            infomsg = 'backend is not installed'
             tst = os.path.basename(item.location[0])
             if not tst in ('test_helper.py', 'test_cppyy.py', 'test_pythonify.py',
-                           'test_datatypes.py', 'test_pythonization.py'):
-                py.test.skip("genreflex is not installed")
+                           'test_cpp11features.py', 'test_datatypes.py',
+                           'test_pythonization.py'):
+                py.test.skip(infomsg)
             import re
             if tst == 'test_pythonify.py' and \
                 not re.search("AppTestPYTHONIFY.test0[1-5]", item.location[2]):
-                py.test.skip("genreflex is not installed")
+                py.test.skip(infomsg)
+            elif tst == 'test_cpp11features.py' and \
+                not re.search("AppTestCPP11FEATURES.test02", item.location[2]):
+                py.test.skip(infomsg)
             elif tst == 'test_datatypes.py' and \
                 not re.search("AppTestDATATYPES.test0[1-7]", item.location[2]):
-                py.test.skip("genreflex is not installed")
+                py.test.skip(infomsg)
             elif tst == 'test_pythonization.py' and \
                 not re.search("AppTestPYTHONIZATION.test0[0]", item.location[2]):
-                py.test.skip("genreflex is not installed")
+                py.test.skip(infomsg)
 
 def pytest_ignore_collect(path, config):
     if py.path.local.sysfind('genreflex') is None and config.option.runappdirect:
diff --git a/pypy/module/_cppyy/test/datatypes.cxx b/pypy/module/_cppyy/test/datatypes.cxx
--- a/pypy/module/_cppyy/test/datatypes.cxx
+++ b/pypy/module/_cppyy/test/datatypes.cxx
@@ -113,6 +113,7 @@
 float                CppyyTestData::get_float()   { return m_float; }
 double               CppyyTestData::get_double()  { return m_double; }
 long double          CppyyTestData::get_ldouble() { return m_ldouble; }
+long double          CppyyTestData::get_ldouble_def(long double ld) { return ld; }
 CppyyTestData::EWhat CppyyTestData::get_enum()    { return m_enum; }
 void*                CppyyTestData::get_voidp()   { return m_voidp; }
 
diff --git a/pypy/module/_cppyy/test/datatypes.h b/pypy/module/_cppyy/test/datatypes.h
--- a/pypy/module/_cppyy/test/datatypes.h
+++ b/pypy/module/_cppyy/test/datatypes.h
@@ -32,6 +32,8 @@
         enum    {E1 = -1};
         enum EE {E2 = -1};
     };
+
+    typedef enum { AA = 1, BB, CC, DD } letter_code;
 }
 
 
@@ -94,6 +96,8 @@
     float                get_float();
     double               get_double();
     long double          get_ldouble();
+    typedef long double aap_t;
+    long double          get_ldouble_def(long double ld = aap_t(1));
     EWhat                get_enum();
     void*                get_voidp();
 
diff --git a/pypy/module/_cppyy/test/datatypes.xml b/pypy/module/_cppyy/test/datatypes.xml
--- a/pypy/module/_cppyy/test/datatypes.xml
+++ b/pypy/module/_cppyy/test/datatypes.xml
@@ -6,6 +6,7 @@
   <enum name="EFruit" />
   <enum name="EnumSpace::E" />
   <class name="EnumSpace::EnumClass" />
+  <enum name="EnumSpace::letter_code" />
 
   <function pattern="get_*" />
   <function pattern="set_*" />
diff --git a/pypy/module/_cppyy/test/pythonizables.cxx b/pypy/module/_cppyy/test/pythonizables.cxx
--- a/pypy/module/_cppyy/test/pythonizables.cxx
+++ b/pypy/module/_cppyy/test/pythonizables.cxx
@@ -27,3 +27,5 @@
 unsigned int pyzables::pass_mine_rp(Countable c) { return c.m_check; }
 unsigned int pyzables::pass_mine_rp_ref(const Countable& c) { return c.m_check; }
 unsigned int pyzables::pass_mine_rp_ptr(const Countable* c) { return c->m_check; }
+
+pyzables::Countable* pyzables::gime_naked_countable() { return new Countable{}; }
diff --git a/pypy/module/_cppyy/test/pythonizables.h b/pypy/module/_cppyy/test/pythonizables.h
--- a/pypy/module/_cppyy/test/pythonizables.h
+++ b/pypy/module/_cppyy/test/pythonizables.h
@@ -57,4 +57,6 @@
 unsigned int pass_mine_rp_ref(const Countable&);
 unsigned int pass_mine_rp_ptr(const Countable*);
 
+Countable* gime_naked_countable();
+
 } // namespace pyzables
diff --git a/pypy/module/_cppyy/test/stltypes.h b/pypy/module/_cppyy/test/stltypes.h
--- a/pypy/module/_cppyy/test/stltypes.h
+++ b/pypy/module/_cppyy/test/stltypes.h
@@ -4,12 +4,23 @@
 #include <utility>
 #include <vector>
 
+
 //- basic example class
 class just_a_class {
 public:
     int m_i;
 };
 

From pypy.commits at gmail.com  Tue Aug  7 17:11:42 2018
From: pypy.commits at gmail.com (arigo)
Date: Tue, 07 Aug 2018 14:11:42 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge heads
Message-ID: <5b6a0b0e.1c69fb81.9058f.c218@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94974:a828a10bfcfd
Date: 2018-08-07 23:10 +0200
http://bitbucket.org/pypy/pypy/changeset/a828a10bfcfd/

Log:	hg merge heads

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -367,8 +367,8 @@
                                        names)
         self._visit_arg_annotations(args.kwonlyargs, names)
         kwarg = args.kwarg
-        if args.kwarg:
-            self._visit_arg_annotation(args.kwarg.arg, args.kwarg.annotation,
+        if kwarg:
+            self._visit_arg_annotation(kwarg.arg, kwarg.annotation,
                                        names)
         self._visit_arg_annotation("return", returns, names)
         l = len(names)
diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -622,6 +622,10 @@
         assert isinstance(args, ast.arguments)
         if args.args:
             self._visit_arg_annotations(args.args)
+        if args.vararg:
+            self._visit_arg_annotation(args.vararg)
+        if args.kwarg:
+            self._visit_arg_annotation(args.kwarg)
         if args.kwonlyargs:
             self._visit_arg_annotations(args.kwonlyargs)
         if func.returns:
@@ -630,8 +634,11 @@
     def _visit_arg_annotations(self, args):
         for arg in args:
             assert isinstance(arg, ast.arg)
-            if arg.annotation:
-                arg.annotation.walkabout(self)
+            self._visit_arg_annotation(arg)
+
+    def _visit_arg_annotation(self, arg):
+        if arg.annotation:
+            arg.annotation.walkabout(self)
 
     def visit_Name(self, name):
         if name.ctx == ast.Load:
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -470,7 +470,7 @@
 
     def test_cmd_co_name(self):
         child = self.spawn(['-c',
-                    'import sys; print sys._getframe(0).f_code.co_name'])
+                    'import sys; print(sys._getframe(0).f_code.co_name)'])
         child.expect('<module>')
 
     def test_ignore_python_inspect(self):
diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py
--- a/pypy/interpreter/test/test_executioncontext.py
+++ b/pypy/interpreter/test/test_executioncontext.py
@@ -149,7 +149,7 @@
         pass
         """)
         space.getexecutioncontext().setllprofile(None, None)
-        assert l == ['call', 'return', 'call', 'return']
+        assert l[-4:] == ['call', 'return', 'call', 'return']
 
     def test_llprofile_c_call(self):
         from pypy.interpreter.function import Function, Method
@@ -173,15 +173,15 @@
             return
             """ % snippet)
             space.getexecutioncontext().setllprofile(None, None)
-            assert l == ['call', 'return', 'call', 'c_call', 'c_return', 'return']
-            if isinstance(seen[0], Method):
-                w_class = space.type(seen[0].w_instance)
+            assert l[-6:] == ['call', 'return', 'call', 'c_call', 'c_return', 'return']
+            if isinstance(seen[-1], Method):
+                w_class = space.type(seen[-1].w_instance)
                 found = 'method %s of %s' % (
-                    seen[0].w_function.name,
+                    seen[-1].w_function.name,
                     w_class.getname(space).encode('utf-8'))
             else:
-                assert isinstance(seen[0], Function)
-                found = 'builtin %s' % seen[0].name
+                assert isinstance(seen[-1], Function)
+                found = 'builtin %s' % seen[-1].name
             assert found == expected_c_call
 
         check_snippet('l = []; l.append(42)', 'method append of list')
@@ -210,7 +210,7 @@
             return
             """ % snippet)
             space.getexecutioncontext().setllprofile(None, None)
-            assert l == ['call', 'return', 'call', 'c_call', 'c_exception', 'return']
+            assert l[-6:] == ['call', 'return', 'call', 'c_call', 'c_exception', 'return']
 
         check_snippet('d = {}; d.__getitem__(42)')
 
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -153,6 +153,8 @@
         r"""
         seen = []
         def tracer(f, event, *args):
+            if f.f_code.co_name == "decode":
+                return tracer
             seen.append((event, f.f_lineno))
             if len(seen) == 5:
                 f.f_lineno = 1       # bug shown only when setting lineno to 1
@@ -297,7 +299,8 @@
 
         l = []
         def trace(a,b,c):
-            l.append((a,b,c))
+            if a.f_code.co_name != "decode":
+                l.append((a,b,c))
 
         def f():
             h = _testing.Hidden()
diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py
--- a/pypy/interpreter/test/test_syntax.py
+++ b/pypy/interpreter/test/test_syntax.py
@@ -691,6 +691,16 @@
                                     "bye" : 5, "kw" : 6, "return" : 42}
         """
 
+    def test_bug_annotations_lambda(self):
+        """
+        # those used to crash
+        def broken(*a: lambda x: None):
+            pass
+
+        def broken(**a: lambda x: None):
+            pass
+        """
+
 class AppTestSyntaxError:
 
     def test_tokenizer_error_location(self):

From pypy.commits at gmail.com  Tue Aug  7 17:47:42 2018
From: pypy.commits at gmail.com (arigo)
Date: Tue, 07 Aug 2018 14:47:42 -0700 (PDT)
Subject: [pypy-commit] cffi default: Merged in
 kelledin/cffi/create-tmp-homedir (pull request #89)
Message-ID: <5b6a137e.1c69fb81.b6e30.b9ec@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: 
Changeset: r3135:f888d26538c0
Date: 2018-08-07 21:47 +0000
http://bitbucket.org/cffi/cffi/changeset/f888d26538c0/

Log:	Merged in kelledin/cffi/create-tmp-homedir (pull request #89)

	point $HOME to empty tempdir rather than a nonexistent path

diff --git a/testing/cffi1/test_zdist.py b/testing/cffi1/test_zdist.py
--- a/testing/cffi1/test_zdist.py
+++ b/testing/cffi1/test_zdist.py
@@ -2,6 +2,8 @@
 import subprocess
 import cffi
 from testing.udir import udir
+from shutil import rmtree
+from tempfile import mkdtemp
 
 
 def chdir_to_tmp(f):
@@ -33,13 +35,20 @@
         env = os.environ.copy()
         # a horrible hack to prevent distutils from finding ~/.pydistutils.cfg
         # (there is the --no-user-cfg option, but not in Python 2.6...)
-        env['HOME'] = '/this/path/does/not/exist'
+        # NOTE: pointing $HOME to a nonexistent directory can break certain things
+        # that look there for configuration (like ccache).
+        tmp_home = mkdtemp()
+        assert tmp_home != None, "cannot create temporary homedir"
+        env['HOME'] = tmp_home
         if cwd is None:
             newpath = self.rootdir
             if 'PYTHONPATH' in env:
                 newpath += os.pathsep + env['PYTHONPATH']
             env['PYTHONPATH'] = newpath
-        subprocess.check_call([self.executable] + args, cwd=cwd, env=env)
+        try:
+            subprocess.check_call([self.executable] + args, cwd=cwd, env=env)
+        finally:
+            rmtree(tmp_home)
 
     def _prepare_setuptools(self):
         if hasattr(TestDist, '_setuptools_ready'):

From pypy.commits at gmail.com  Tue Aug  7 17:47:45 2018
From: pypy.commits at gmail.com (kelledin)
Date: Tue, 07 Aug 2018 14:47:45 -0700 (PDT)
Subject: [pypy-commit] cffi create-tmp-homedir: point $HOME to empty tempdir
 rather than a nonexistent path
Message-ID: <5b6a1381.1c69fb81.10d84.d018@mx.google.com>

Author: Frank Schaefer <kelledin at gmail.com>
Branch: create-tmp-homedir
Changeset: r3134:916787e62a47
Date: 2018-08-07 15:30 -0500
http://bitbucket.org/cffi/cffi/changeset/916787e62a47/

Log:	point $HOME to empty tempdir rather than a nonexistent path

diff --git a/testing/cffi1/test_zdist.py b/testing/cffi1/test_zdist.py
--- a/testing/cffi1/test_zdist.py
+++ b/testing/cffi1/test_zdist.py
@@ -2,6 +2,8 @@
 import subprocess
 import cffi
 from testing.udir import udir
+from shutil import rmtree
+from tempfile import mkdtemp
 
 
 def chdir_to_tmp(f):
@@ -33,13 +35,20 @@
         env = os.environ.copy()
         # a horrible hack to prevent distutils from finding ~/.pydistutils.cfg
         # (there is the --no-user-cfg option, but not in Python 2.6...)
-        env['HOME'] = '/this/path/does/not/exist'
+        # NOTE: pointing $HOME to a nonexistent directory can break certain things
+        # that look there for configuration (like ccache).
+        tmp_home = mkdtemp()
+        assert tmp_home != None, "cannot create temporary homedir"
+        env['HOME'] = tmp_home
         if cwd is None:
             newpath = self.rootdir
             if 'PYTHONPATH' in env:
                 newpath += os.pathsep + env['PYTHONPATH']
             env['PYTHONPATH'] = newpath
-        subprocess.check_call([self.executable] + args, cwd=cwd, env=env)
+        try:
+            subprocess.check_call([self.executable] + args, cwd=cwd, env=env)
+        finally:
+            rmtree(tmp_home)
 
     def _prepare_setuptools(self):
         if hasattr(TestDist, '_setuptools_ready'):

From pypy.commits at gmail.com  Wed Aug  8 05:28:42 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 Aug 2018 02:28:42 -0700 (PDT)
Subject: [pypy-commit] cffi default: msvc support
Message-ID: <5b6ab7ca.1c69fb81.1fab0.f57c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3136:ce24777ce311
Date: 2018-08-08 11:28 +0200
http://bitbucket.org/cffi/cffi/changeset/ce24777ce311/

Log:	msvc support

diff --git a/c/misc_thread_common.h b/c/misc_thread_common.h
--- a/c/misc_thread_common.h
+++ b/c/misc_thread_common.h
@@ -44,11 +44,13 @@
        PyThreadState any more, which are all supposed to be freed anyway
        very soon after the present cffi_tls_shutdown() function is called.
      */
+    PyObject *ofn;
+
     TLS_DEL_LOCK();
     cffi_tls_delete = 0;   /* Py_Finalize() called */
     TLS_DEL_UNLOCK();
 
-    PyObject *ofn = old_exitfunc;
+    ofn = old_exitfunc;
     if (ofn == NULL)
     {
         Py_INCREF(Py_None);

From pypy.commits at gmail.com  Wed Aug  8 05:46:50 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 Aug 2018 02:46:50 -0700 (PDT)
Subject: [pypy-commit] cffi default: Issue #375
Message-ID: <5b6abc0a.1c69fb81.4a2e0.8fed@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3137:1dd7bdcf4993
Date: 2018-08-08 11:44 +0200
http://bitbucket.org/cffi/cffi/changeset/1dd7bdcf4993/

Log:	Issue #375

	Port another test from CPython, which also passes already on win64.

diff --git a/testing/cffi0/test_ownlib.py b/testing/cffi0/test_ownlib.py
--- a/testing/cffi0/test_ownlib.py
+++ b/testing/cffi0/test_ownlib.py
@@ -102,6 +102,11 @@
 {
     return (unsigned int)(a + 42);
 }
+
+EXPORT void modify_struct_value(RECT r)
+{
+    r.left = r.right = r.top = r.bottom = 500;
+}
 """
 
 class TestOwnLib(object):
@@ -330,3 +335,25 @@
         assert lib.foo_2bytes(u+'\u1234') == u+'\u125e'
         assert lib.foo_4bytes(u+'\u1234') == u+'\u125e'
         assert lib.foo_4bytes(u+'\U00012345') == u+'\U0001236f'
+
+    def test_modify_struct_value(self):
+        if self.module is None:
+            py.test.skip("fix the auto-generation of the tiny test lib")
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("""
+            typedef struct {
+                long left;
+                long top;
+                long right;
+                long bottom;
+            } RECT;
+
+            void modify_struct_value(RECT r);
+        """)
+        lib = ffi.dlopen(self.module)
+        s = ffi.new("RECT *", [11, 22, 33, 44])
+        lib.modify_struct_value(s[0])
+        assert s.left == 11
+        assert s.top == 22
+        assert s.right == 33
+        assert s.bottom == 44

From pypy.commits at gmail.com  Wed Aug  8 07:11:42 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 Aug 2018 04:11:42 -0700 (PDT)
Subject: [pypy-commit] pypy default: update to cffi/1dd7bdcf4993
Message-ID: <5b6acfee.1c69fb81.cda0e.5f40@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94978:a233de8fbd4b
Date: 2018-08-08 13:10 +0200
http://bitbucket.org/pypy/pypy/changeset/a233de8fbd4b/

Log:	update to cffi/1dd7bdcf4993

diff --git a/lib_pypy/cffi/_cffi_errors.h b/lib_pypy/cffi/_cffi_errors.h
--- a/lib_pypy/cffi/_cffi_errors.h
+++ b/lib_pypy/cffi/_cffi_errors.h
@@ -50,7 +50,9 @@
         "import sys\n"
         "class FileLike:\n"
         "  def write(self, x):\n"
-        "    of.write(x)\n"
+        "    try:\n"
+        "      of.write(x)\n"
+        "    except: pass\n"
         "    self.buf += x\n"
         "fl = FileLike()\n"
         "fl.buf = ''\n"
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1946,3 +1946,30 @@
         # only works with the Python FFI instances
         ffi = FFI(backend=self.Backend())
         assert ffi.sizeof("struct{int a;}") == ffi.sizeof("int")
+
+    def test_callback_large_struct(self):
+        ffi = FFI(backend=self.Backend())
+        # more than 8 bytes
+        ffi.cdef("struct foo_s { unsigned long a, b, c; };")
+        #
+        @ffi.callback("void(struct foo_s)")
+        def cb(s):
+            seen.append(ffi.typeof(s))
+            s.a += 1
+            s.b += 2
+            s.c += 3
+            seen.append(s.a)
+            seen.append(s.b)
+            seen.append(s.c)
+        #
+        s1 = ffi.new("struct foo_s *", {'a': 100, 'b': 200, 'c': 300})
+        seen = []
+        cb(s1[0])
+        assert len(seen) == 4
+        assert s1.a == 100     # unmodified
+        assert s1.b == 200
+        assert s1.c == 300
+        assert seen[0] == ffi.typeof("struct foo_s")
+        assert seen[1] == 101
+        assert seen[2] == 202
+        assert seen[3] == 303
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
@@ -3,6 +3,8 @@
 import subprocess
 import cffi
 from pypy.module.test_lib_pypy.cffi_tests.udir import udir
+from shutil import rmtree
+from tempfile import mkdtemp
 
 
 def chdir_to_tmp(f):
@@ -34,13 +36,20 @@
         env = os.environ.copy()
         # a horrible hack to prevent distutils from finding ~/.pydistutils.cfg
         # (there is the --no-user-cfg option, but not in Python 2.6...)
-        env['HOME'] = '/this/path/does/not/exist'
+        # NOTE: pointing $HOME to a nonexistent directory can break certain things
+        # that look there for configuration (like ccache).
+        tmp_home = mkdtemp()
+        assert tmp_home != None, "cannot create temporary homedir"
+        env['HOME'] = tmp_home
         if cwd is None:
             newpath = self.rootdir
             if 'PYTHONPATH' in env:
                 newpath += os.pathsep + env['PYTHONPATH']
             env['PYTHONPATH'] = newpath
-        subprocess.check_call([self.executable] + args, cwd=cwd, env=env)
+        try:
+            subprocess.check_call([self.executable] + args, cwd=cwd, env=env)
+        finally:
+            rmtree(tmp_home)
 
     def _prepare_setuptools(self):
         if hasattr(TestDist, '_setuptools_ready'):

From pypy.commits at gmail.com  Wed Aug  8 07:16:00 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 Aug 2018 04:16:00 -0700 (PDT)
Subject: [pypy-commit] pypy default: update to cffi/1dd7bdcf4993 (really
 this time)
Message-ID: <5b6ad0f0.1c69fb81.49f87.025f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94979:1a106114b2a3
Date: 2018-08-08 13:14 +0200
http://bitbucket.org/pypy/pypy/changeset/1a106114b2a3/

Log:	update to cffi/1dd7bdcf4993 (really this time)

diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
@@ -103,6 +103,11 @@
 {
     return (unsigned int)(a + 42);
 }
+
+EXPORT void modify_struct_value(RECT r)
+{
+    r.left = r.right = r.top = r.bottom = 500;
+}
 """
 
 class TestOwnLib(object):
@@ -331,3 +336,25 @@
         assert lib.foo_2bytes(u+'\u1234') == u+'\u125e'
         assert lib.foo_4bytes(u+'\u1234') == u+'\u125e'
         assert lib.foo_4bytes(u+'\U00012345') == u+'\U0001236f'
+
+    def test_modify_struct_value(self):
+        if self.module is None:
+            py.test.skip("fix the auto-generation of the tiny test lib")
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("""
+            typedef struct {
+                long left;
+                long top;
+                long right;
+                long bottom;
+            } RECT;
+
+            void modify_struct_value(RECT r);
+        """)
+        lib = ffi.dlopen(self.module)
+        s = ffi.new("RECT *", [11, 22, 33, 44])
+        lib.modify_struct_value(s[0])
+        assert s.left == 11
+        assert s.top == 22
+        assert s.right == 33
+        assert s.bottom == 44

From pypy.commits at gmail.com  Wed Aug  8 11:17:18 2018
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 08 Aug 2018 08:17:18 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix issue with empty string as module
 attribute.
Message-ID: <5b6b097e.1c69fb81.d5426.ef24@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r94980:8c0c734e9e02
Date: 2018-08-08 15:59 +0100
http://bitbucket.org/pypy/pypy/changeset/8c0c734e9e02/

Log:	Fix issue with empty string as module attribute.

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1628,7 +1628,7 @@
         else:
             skip_leading_underscores = False
         for name in all:
-            if skip_leading_underscores and name[0]=='_':
+            if skip_leading_underscores and name and name[0] == '_':
                 continue
             into_locals[name] = getattr(module, name)
 ''', filename=__file__)
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -69,8 +69,8 @@
              foobar    = "found = 123",
              barbaz    = "other = 543")
     setuppkg("pkg.withoutall",
-             __init__  = "",
-             foobar    = "found = 123")
+             __init__  = "globals()[''] = 456",
+             foobar    = "found = 123\n")
     setuppkg("pkg.bogusall",
              __init__  = "__all__ = 42")
     setuppkg("pkg_r", inpkg = "import x.y")
@@ -373,7 +373,7 @@
         raises(ImportError, __import__, 'xxxbadmodule', fromlist=[u'xx'])
         mod = __import__('collections', fromlist=[u'defaultdict'])
         assert mod is not None
-        
+
 
     def test_import_relative_back_to_absolute2(self):
         from pkg import abs_x_y
@@ -745,6 +745,13 @@
             exec "from pkg.withoutall import *" in d
             assert d["foobar"].found == 123
 
+    def test_import_star_empty_string(self):
+        for case in ["not-imported-yet", "already-imported"]:
+            d = {}
+            exec "from pkg.withoutall import *" in d
+            assert "" in d
+
+
     def test_import_star_with_bogus___all__(self):
         for case in ["not-imported-yet", "already-imported"]:
             try:

From pypy.commits at gmail.com  Thu Aug  9 02:40:28 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 08 Aug 2018 23:40:28 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix thread => _thread
Message-ID: <5b6be1dc.1c69fb81.314d1.1986@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94981:103f347f34e8
Date: 2018-08-09 08:39 +0200
http://bitbucket.org/pypy/pypy/changeset/103f347f34e8/

Log:	Fix thread => _thread

diff --git a/lib_pypy/grp.py b/lib_pypy/grp.py
--- a/lib_pypy/grp.py
+++ b/lib_pypy/grp.py
@@ -5,8 +5,8 @@
 import os
 from _pwdgrp_cffi import ffi, lib
 import _structseq
-import thread
-_lock = thread.allocate_lock()
+import _thread
+_lock = _thread.allocate_lock()
 
 try: from __pypy__ import builtinify
 except ImportError: builtinify = lambda f: f
diff --git a/lib_pypy/pwd.py b/lib_pypy/pwd.py
--- a/lib_pypy/pwd.py
+++ b/lib_pypy/pwd.py
@@ -12,8 +12,8 @@
 
 from _pwdgrp_cffi import ffi, lib
 import _structseq
-import thread
-_lock = thread.allocate_lock()
+import _thread
+_lock = _thread.allocate_lock()
 
 try: from __pypy__ import builtinify
 except ImportError: builtinify = lambda f: f

From pypy.commits at gmail.com  Thu Aug  9 09:47:11 2018
From: pypy.commits at gmail.com (arigo)
Date: Thu, 09 Aug 2018 06:47:11 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2865
Message-ID: <5b6c45df.1c69fb81.8d83c.08f8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94982:95232deededb
Date: 2018-08-09 15:46 +0200
http://bitbucket.org/pypy/pypy/changeset/95232deededb/

Log:	Issue #2865

	Make types.MemberDescriptorType be the type of app-level slots

diff --git a/lib-python/2.7/types.py b/lib-python/2.7/types.py
--- a/lib-python/2.7/types.py
+++ b/lib-python/2.7/types.py
@@ -83,9 +83,19 @@
 DictProxyType = type(TypeType.__dict__)
 NotImplementedType = type(NotImplemented)
 
-# For Jython, the following two types are identical
+#
+# On CPython, FunctionType.__code__ is a 'getset_descriptor', but
+# FunctionType.__globals__ is a 'member_descriptor', just like app-level
+# slots.  On PyPy, all descriptors of built-in types are
+# 'getset_descriptor', but the app-level slots are 'member_descriptor'
+# as well.  (On Jython the situation might still be different.)
+#
+# Note that MemberDescriptorType was equal to GetSetDescriptorType in
+# PyPy <= 6.0.
+#
 GetSetDescriptorType = type(FunctionType.func_code)
-MemberDescriptorType = type(FunctionType.func_globals)
+class _C(object): __slots__ = 's'
+MemberDescriptorType = type(_C.s)
 
 del sys, _f, _g, _C, _x                           # Not for export
 

From pypy.commits at gmail.com  Thu Aug  9 10:33:51 2018
From: pypy.commits at gmail.com (arigo)
Date: Thu, 09 Aug 2018 07:33:51 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default (and port the changes
 from 2.7/types.py to 3/types.py)
Message-ID: <5b6c50cf.1c69fb81.3c6e3.027d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94983:f938ea58f52c
Date: 2018-08-09 16:32 +0200
http://bitbucket.org/pypy/pypy/changeset/f938ea58f52c/

Log:	hg merge default (and port the changes from 2.7/types.py to
	3/types.py)

diff --git a/lib-python/3/types.py b/lib-python/3/types.py
--- a/lib-python/3/types.py
+++ b/lib-python/3/types.py
@@ -41,9 +41,19 @@
     FrameType = type(tb.tb_frame)
     tb = None; del tb
 
-# For Jython, the following two types are identical
+#
+# On CPython, FunctionType.__code__ is a 'getset_descriptor', but
+# FunctionType.__globals__ is a 'member_descriptor', just like app-level
+# slots.  On PyPy, all descriptors of built-in types are
+# 'getset_descriptor', but the app-level slots are 'member_descriptor'
+# as well.  (On Jython the situation might still be different.)
+#
+# Note that MemberDescriptorType was equal to GetSetDescriptorType in
+# PyPy <= 6.0.
+#
 GetSetDescriptorType = type(FunctionType.__code__)
-MemberDescriptorType = type(FunctionType.__globals__)
+class _C: __slots__ = 's'
+MemberDescriptorType = type(_C.s)
 
 del sys, _f, _g, _C, _c,                           # Not for export
 
diff --git a/lib_pypy/cffi/_cffi_errors.h b/lib_pypy/cffi/_cffi_errors.h
--- a/lib_pypy/cffi/_cffi_errors.h
+++ b/lib_pypy/cffi/_cffi_errors.h
@@ -50,7 +50,9 @@
         "import sys\n"
         "class FileLike:\n"
         "  def write(self, x):\n"
-        "    of.write(x)\n"
+        "    try:\n"
+        "      of.write(x)\n"
+        "    except: pass\n"
         "    self.buf += x\n"
         "fl = FileLike()\n"
         "fl.buf = ''\n"
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -2076,7 +2076,7 @@
         else:
             skip_leading_underscores = False
         for name in all:
-            if skip_leading_underscores and name[0]=='_':
+            if skip_leading_underscores and name and name[0] == '_':
                 continue
             into_locals[name] = getattr(module, name)
 ''', filename=__file__)
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -101,8 +101,8 @@
              foobar    = "found = 123",
              barbaz    = "other = 543")
     setuppkg("pkg.withoutall",
-             __init__  = "",
-             foobar    = "found = 123")
+             __init__  = "globals()[''] = 456",
+             foobar    = "found = 123\n")
     setuppkg("pkg.bogusall",
              __init__  = "__all__ = 42")
     setuppkg("pkg_r", inpkg = "import x.y")
@@ -703,6 +703,13 @@
             exec("from pkg.withoutall import *", d)
             assert d["foobar"].found == 123
 
+    def test_import_star_empty_string(self):
+        for case in ["not-imported-yet", "already-imported"]:
+            d = {}
+            exec("from pkg.withoutall import *", d)
+            assert "" in d
+
+
     def test_import_star_with_bogus___all__(self):
         for case in ["not-imported-yet", "already-imported"]:
             try:
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1946,3 +1946,30 @@
         # only works with the Python FFI instances
         ffi = FFI(backend=self.Backend())
         assert ffi.sizeof("struct{int a;}") == ffi.sizeof("int")
+
+    def test_callback_large_struct(self):
+        ffi = FFI(backend=self.Backend())
+        # more than 8 bytes
+        ffi.cdef("struct foo_s { unsigned long a, b, c; };")
+        #
+        @ffi.callback("void(struct foo_s)")
+        def cb(s):
+            seen.append(ffi.typeof(s))
+            s.a += 1
+            s.b += 2
+            s.c += 3
+            seen.append(s.a)
+            seen.append(s.b)
+            seen.append(s.c)
+        #
+        s1 = ffi.new("struct foo_s *", {'a': 100, 'b': 200, 'c': 300})
+        seen = []
+        cb(s1[0])
+        assert len(seen) == 4
+        assert s1.a == 100     # unmodified
+        assert s1.b == 200
+        assert s1.c == 300
+        assert seen[0] == ffi.typeof("struct foo_s")
+        assert seen[1] == 101
+        assert seen[2] == 202
+        assert seen[3] == 303
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
@@ -103,6 +103,11 @@
 {
     return (unsigned int)(a + 42);
 }
+
+EXPORT void modify_struct_value(RECT r)
+{
+    r.left = r.right = r.top = r.bottom = 500;
+}
 """
 
 class TestOwnLib(object):
@@ -331,3 +336,25 @@
         assert lib.foo_2bytes(u+'\u1234') == u+'\u125e'
         assert lib.foo_4bytes(u+'\u1234') == u+'\u125e'
         assert lib.foo_4bytes(u+'\U00012345') == u+'\U0001236f'
+
+    def test_modify_struct_value(self):
+        if self.module is None:
+            py.test.skip("fix the auto-generation of the tiny test lib")
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("""
+            typedef struct {
+                long left;
+                long top;
+                long right;
+                long bottom;
+            } RECT;
+
+            void modify_struct_value(RECT r);
+        """)
+        lib = ffi.dlopen(self.module)
+        s = ffi.new("RECT *", [11, 22, 33, 44])
+        lib.modify_struct_value(s[0])
+        assert s.left == 11
+        assert s.top == 22
+        assert s.right == 33
+        assert s.bottom == 44
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
@@ -3,6 +3,8 @@
 import subprocess
 import cffi
 from pypy.module.test_lib_pypy.cffi_tests.udir import udir
+from shutil import rmtree
+from tempfile import mkdtemp
 
 
 def chdir_to_tmp(f):
@@ -34,13 +36,20 @@
         env = os.environ.copy()
         # a horrible hack to prevent distutils from finding ~/.pydistutils.cfg
         # (there is the --no-user-cfg option, but not in Python 2.6...)
-        env['HOME'] = '/this/path/does/not/exist'
+        # NOTE: pointing $HOME to a nonexistent directory can break certain things
+        # that look there for configuration (like ccache).
+        tmp_home = mkdtemp()
+        assert tmp_home != None, "cannot create temporary homedir"
+        env['HOME'] = tmp_home
         if cwd is None:
             newpath = self.rootdir
             if 'PYTHONPATH' in env:
                 newpath += os.pathsep + env['PYTHONPATH']
             env['PYTHONPATH'] = newpath
-        subprocess.check_call([self.executable] + args, cwd=cwd, env=env)
+        try:
+            subprocess.check_call([self.executable] + args, cwd=cwd, env=env)
+        finally:
+            rmtree(tmp_home)
 
     def _prepare_setuptools(self):
         if hasattr(TestDist, '_setuptools_ready'):

From pypy.commits at gmail.com  Thu Aug  9 16:30:43 2018
From: pypy.commits at gmail.com (mattip)
Date: Thu, 09 Aug 2018 13:30:43 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: add a lgt arg to newtext,
 change error _compute_value accordingly
Message-ID: <5b6ca473.1c69fb81.44c37.d02f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94984:07a4929a661d
Date: 2018-08-09 13:27 -0700
http://bitbucket.org/pypy/pypy/changeset/07a4929a661d/

Log:	add a lgt arg to newtext, change error _compute_value accordingly

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -9,7 +9,7 @@
 from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.rlib.objectmodel import dont_inline, not_rpython
 from rpython.rlib import rstack, rstackovf
-from rpython.rlib import rwin32
+from rpython.rlib import rwin32, runicode
 
 from pypy.interpreter import debug
 
@@ -71,7 +71,7 @@
         space = getattr(self.w_type, 'space', None)
         if space is not None:
             if self.__class__ is not OperationError and s is None:
-                s = self._compute_value(space)
+                s, lgt = self._compute_value(space)
             try:
                 s = space.text_w(s)
             except Exception:
@@ -305,8 +305,8 @@
     def get_w_value(self, space):
         w_value = self._w_value
         if w_value is None:
-            value = self._compute_value(space)
-            self._w_value = w_value = space.newtext(value)
+            value, lgt = self._compute_value(space)
+            self._w_value = w_value = space.newtext(value, lgt)
         return w_value
 
     def _compute_value(self, space):
@@ -477,10 +477,10 @@
     if isinstance(string, unicode):
         return string
     assert isinstance(string, str)
-    return string.decode('utf8')
-    #result, consumed = runicode.str_decode_utf_8(
-    #    string, len(string), "replace", final=True)
-    #return result
+    #return string.decode('utf8')
+    result, consumed = runicode.str_decode_utf_8(
+        string, len(string), "replace", final=True)
+    return result
 
 def get_operrcls2(valuefmt):
     valuefmt = valuefmt.decode('ascii')
@@ -502,6 +502,7 @@
                 self.setup(w_type)
 
             def _compute_value(self, space):
+                # TODO: avoid utf8->unicode->utf8 dance
                 lst = [None] * (len(formats) + len(formats) + 1)
                 for i, fmt, attr in entries:
                     lst[i + i] = self.xstrings[i]
@@ -523,7 +524,8 @@
                     elif fmt == '8':
                         # u'str\uxxxx' -> 'str\xXX\xXX' -> u"'str\xXX\xXX'"
                         if isinstance(value, unicode):
-                            result = value.encode('utf8')
+                            result = runicode.unicode_encode_utf_8(value,
+                                     len(value), 'strict', allow_surrogates=True)
                         else:
                             from pypy.interpreter import unicodehelper
                             result = _decode_utf8(unicodehelper.str_decode_utf8(
@@ -536,7 +538,12 @@
                             result = _decode_utf8(str(value))
                     lst[i + i + 1] = result
                 lst[-1] = self.xstrings[-1]
-                return u''.join(lst)
+                retval = u''.join(lst)
+                # We need to annotate both allow_surrogates=True,False
+                # since this function is used to replace uni.encode('utf8')
+                # deep in rpython
+                return runicode.unicode_encode_utf_8(retval, len(retval),
+                             'strict', allow_surrogates=False), len(retval)
         #
         _fmtcache2[formats] = OpErrFmt
     return OpErrFmt, strings
@@ -547,7 +554,7 @@
         self.setup(w_type)
 
     def _compute_value(self, space):
-        return self._value.decode('utf-8')
+        return self._value, len(self._value)
 
     def async(self, space):
         # also matches a RuntimeError("maximum rec.") if the stack is
@@ -639,7 +646,7 @@
             msg = u'Windows Error %d' % winerror
         w_errno = space.w_None
         w_winerror = space.newint(winerror)
-        w_msg = space.newtext(msg)
+        w_msg = space.newtext(msg.encode('utf8'), len(msg))
     else:
         errno = e.errno
         if errno == EINTR:
@@ -653,7 +660,7 @@
             msg = u'error %d' % errno
         w_errno = space.newint(errno)
         w_winerror = space.w_None
-        w_msg = space.newtext(msg)
+        w_msg = space.newtext(msg.encode('utf8'), len(msg))
 
     if w_filename is None:
         w_filename = space.w_None
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -1122,7 +1122,7 @@
             kw_defs_w = []
             for name, w_def in sorted(alldefs_w.items()):
                 assert name in sig.kwonlyargnames
-                w_name = space.newtext(name.decode('utf-8'))
+                w_name = space.newtext(name)
                 kw_defs_w.append((w_name, w_def))
 
         return defs_w, kw_defs_w
diff --git a/pypy/interpreter/pyparser/error.py b/pypy/interpreter/pyparser/error.py
--- a/pypy/interpreter/pyparser/error.py
+++ b/pypy/interpreter/pyparser/error.py
@@ -46,7 +46,7 @@
             if len(self.text) != offset:
                 text, _ = str_decode_utf_8_impl(self.text, len(self.text),
                              'replace', False, replace_error_handler, True)
-            w_text = space.newtext(text)
+            w_text = space.newtext(text.encode('utf8'), len(text))
         return space.newtuple([
             space.newtext(self.msg),
             space.newtuple([
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -92,7 +92,7 @@
     def getitem(self, obj, key):
         return obj[key]
 
-    def wrap(self, obj):
+    def wrap(self, obj, lgt=-1):
         return obj
     newtext = wrap
 
diff --git a/pypy/interpreter/test/test_error.py b/pypy/interpreter/test/test_error.py
--- a/pypy/interpreter/test/test_error.py
+++ b/pypy/interpreter/test/test_error.py
@@ -133,7 +133,7 @@
         w_OSError = [OSError]
         w_EnvironmentError = [EnvironmentError]
         w_None = None
-        def wrap(self, obj):
+        def wrap(self, obj, lgt=-1):
             return [obj]
         newint = newtext = newfilename = wrap
         def call_function(self, exc, w_errno, w_msg, w_filename=None, *args):
diff --git a/pypy/interpreter/test/test_fsencode.py b/pypy/interpreter/test/test_fsencode.py
--- a/pypy/interpreter/test/test_fsencode.py
+++ b/pypy/interpreter/test/test_fsencode.py
@@ -70,7 +70,7 @@
             strs.append(self.special_char)
         for st in strs:
             # check roundtrip
-            w_st = space.newtext(st)
+            w_st = space.newtext(st.encode('utf8'), len(st))
             w_enc = space.fsencode(w_st)
             w_st2 = space.fsdecode(w_enc)
             assert space.eq_w(w_st, w_st2)
@@ -81,7 +81,8 @@
 
     def test_null_byte(self):
         space = self.space
-        w_u = space.newtext(u'abc\x00def')
+        uni = u'abc\x00def'
+        w_u = space.newtext(uni.encode('utf8'), len(uni))
         # this can behave in two different ways depending on how
         # much initialized the space is: space.fsencode() can raise
         # ValueError directly, or return a wrapped bytes with the 0
@@ -94,7 +95,7 @@
         if self.special_char:
             strs.append(self.special_char)
         for st in strs:
-            w_st = space.newtext(st)
+            w_st = space.newtext(st.encode('utf8'), len(st))
             w_enc = space.fsencode(w_st)
             space.appexec([w_st, w_enc], """(u, s):
                 import __pypy__
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -95,7 +95,8 @@
         return space.call_method(w_string, 'decode',
                                  getfilesystemencoding(space),
                                  space.newtext('surrogateescape'))
-    return space.newtext(uni)
+    return space.newtext(runicode.unicode_encode_utf_8(uni,
+                                 len(uni), 'strict', allow_surrogates=True), len(uni))
 
 def fsencode(space, w_uni):
     from pypy.module._codecs import interp_codecs
@@ -373,7 +374,7 @@
                 if not final:
                     pos -= 1
                     break
-                r, pos, lgt = errorhandler(errors, "utf8", "unexpected end of data",
+                r, pos = errorhandler(errors, "utf8", "unexpected end of data",
                     s, pos - 1, pos + 1)
                 res.append(r)
                 continue
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -50,9 +50,9 @@
                 length = len(input)
             else:
                 w_cls = space.w_UnicodeEncodeError
-                length = len(input)
-                assert isinstance(input, unicode)
-                w_input = space.newtext((input.encode('utf8'), length, length))
+                assert isinstance(input, str)
+                length = rutf8.codepoints_in_utf8(input)
+                w_input = space.newtext(input, length)
             w_exc =  space.call_function(
                 w_cls,
                 space.newtext(encoding),
@@ -441,7 +441,7 @@
             ch = 0
         if ch == 0:
             raise OperationError(space.type(w_exc), w_exc)
-        return space.newtuple([space.newtext(unichr(ch)),
+        return space.newtuple([space.newtext(unichr(ch).encode('utf8'), 1),
                                space.newint(start + bytelength)])
     else:
         raise oefmt(space.w_TypeError,
@@ -480,7 +480,7 @@
         if not consumed:
             # codec complained about ASCII byte.
             raise OperationError(space.type(w_exc), w_exc)
-        return space.newtuple([space.newtext(replace),
+        return space.newtuple([space.newtext(replace.encode('utf8'), len(replace)),
                                space.newint(start + consumed)])
     else:
         raise oefmt(space.w_TypeError,
@@ -723,9 +723,6 @@
     if errors is None:
         errors = 'strict'
     state = space.fromcache(CodecState)
-    #result = runicode.unicode_encode_utf_8_impl(
-    #    utf8, lgt, errors, state.encode_error_handler,
-    #    allow_surrogates=False)
     result = unicodehelper.utf8_encode_utf_8(utf8, errors,
                      state.encode_error_handler, allow_surrogates=False)
     return space.newtuple([space.newbytes(result), space.newint(lgt)])
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -41,7 +41,8 @@
         if isinstance(ctx, rsre_core.StrMatchContext):
             return space.newbytes(ctx._string[start:end])
         elif isinstance(ctx, rsre_core.UnicodeMatchContext):
-            return space.newtext(ctx._unicodestr[start:end])
+            uni = ctx._unicodestr[start:end]
+            return space.newtext(uni.encode('utf8'), len(uni))
         else:
             # unreachable
             raise SystemError
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -218,8 +218,7 @@
     def newutf8(self, x, l):
         return w_some_obj()
 
-    @specialize.argtype(1)
-    def newtext(self, x):
+    def newtext(self, x, lgt=-1):
         return w_some_obj()
     newtext_or_none = newtext
     newfilename = newtext
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -381,27 +381,22 @@
         return W_BytearrayObject(l)
 
     @specialize.argtype(1)
-    def newtext(self, s):
+    def newtext(self, s, lgt=-1):
         if isinstance(s, unicode):
             s, lgt = s.encode('utf8'), len(s)
-        elif isinstance(s, str):
-            s, lgt, codepoints = decode_utf8sp(self, s)
+        elif isinstance(s, str) and lgt < 0:
+            lgt = rutf8.codepoints_in_utf8(s)
         elif isinstance(s, tuple):
             # result of decode_utf8
             s, lgt, codepoints = s
-        else:
-            # XXX what is s ?
-            lgt = rutf8.check_utf8(s, True)
         assert isinstance(s, str)
         return W_UnicodeObject(s, lgt)
 
-    def newtext_or_none(self, s):
+    def newtext_or_none(self, s, lgt=-1):
         if s is None:
             return self.w_None
-        return self.newtext(s)
+        return self.newtext(s, lgt)
 
-    # XXX find where length is annotated as negative int
-    #@signature(types.any(), types.str(), types.int_nonneg(), returns=types.any())
     def newutf8(self, utf8s, length):
         assert isinstance(utf8s, str)
         return W_UnicodeObject(utf8s, length)
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -271,13 +271,13 @@
         return w_new
 
     def descr_repr(self, space):
-        return space.newtext(_repr_function(self._utf8))
+        return space.newtext(_repr_function(self._utf8)) # quotes=True
 
     def descr_str(self, space):
         if space.is_w(space.type(self), space.w_unicode):
             return self
         # Subtype -- return genuine unicode string with the same value.
-        return space.newtext(space.utf8_w(self))
+        return space.newtext(space.utf8_w(self), space.len_w(self))
 
     def descr_hash(self, space):
         x = compute_hash(self._utf8)
@@ -343,7 +343,7 @@
     def _parse_format_arg(self, space, w_kwds, __args__):
         for i in range(len(__args__.keywords)):
             try:     # pff
-                arg = __args__.keywords[i].decode('utf-8')
+                arg = __args__.keywords[i]
             except UnicodeDecodeError:
                 continue   # uh, just skip that
             space.setitem(w_kwds, space.newtext(arg),

From pypy.commits at gmail.com  Thu Aug  9 16:30:45 2018
From: pypy.commits at gmail.com (mattip)
Date: Thu, 09 Aug 2018 13:30:45 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: replace utf8 with W_Unicode ,
 saves a conversion or two
Message-ID: <5b6ca475.1c69fb81.d90c6.6062@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94985:f6f71b76311e
Date: 2018-08-09 13:28 -0700
http://bitbucket.org/pypy/pypy/changeset/f6f71b76311e/

Log:	replace utf8 with W_Unicode , saves a conversion or two

diff --git a/pypy/interpreter/astcompiler/fstring.py b/pypy/interpreter/astcompiler/fstring.py
--- a/pypy/interpreter/astcompiler/fstring.py
+++ b/pypy/interpreter/astcompiler/fstring.py
@@ -3,6 +3,7 @@
 from pypy.interpreter import error
 from pypy.interpreter import unicodehelper
 from rpython.rlib.rstring import StringBuilder
+from rpython.rlib.rutf8 import codepoints_in_utf8
 
 
 def add_constant_string(astbuilder, joined_pieces, w_string, atom_node):
@@ -21,10 +22,8 @@
     joined_pieces.append(node(w_string, atom_node.get_lineno(),
                                         atom_node.get_column()))
 
-def f_constant_string(astbuilder, joined_pieces, u, atom_node):
-    space = astbuilder.space
-    add_constant_string(astbuilder, joined_pieces, space.newtext(u),
-                        atom_node)
+def f_constant_string(astbuilder, joined_pieces, w_u, atom_node):
+    add_constant_string(astbuilder, joined_pieces, w_u, atom_node)
 
 def f_string_compile(astbuilder, source, atom_node):
     # Note: a f-string is kept as a single literal up to here.
@@ -259,19 +258,20 @@
         i += 1
 
     fstr.current_index = i
+    space = astbuilder.space
     literal = builder.build()
+    lgt = codepoints_in_utf8(literal)
     if not fstr.raw_mode and '\\' in literal:
-        space = astbuilder.space
         literal = parsestring.decode_unicode_utf8(space, literal, 0,
                                                   len(literal))
-        literal, lgt, _ = unicodehelper.decode_unicode_escape(space, literal)
-    return literal.decode('utf-8')
+        literal, pos, lgt = unicodehelper.decode_unicode_escape(space, literal)
+    return space.newtext(literal, lgt)
 
 
 def fstring_find_literal_and_expr(astbuilder, fstr, atom_node, rec):
-    # Return a tuple with the next literal part, and optionally the
+    # Return a tuple with the next literal part as a W_Unicode, and optionally the
     # following expression node.  Updates the current index inside 'fstr'.
-    literal = fstring_find_literal(astbuilder, fstr, atom_node, rec)
+    w_u = fstring_find_literal(astbuilder, fstr, atom_node, rec)
 
     s = fstr.unparsed
     i = fstr.current_index
@@ -283,7 +283,7 @@
         # We must now be the start of an expression, on a '{'.
         assert s[i] == '{'
         expr = fstring_find_expr(astbuilder, fstr, atom_node, rec)
-    return literal, expr
+    return w_u, expr
 
 
 def parse_f_string(astbuilder, joined_pieces, fstr, atom_node, rec=0):
@@ -302,11 +302,11 @@
             "really the case", atom_node)
 
     while True:
-        literal, expr = fstring_find_literal_and_expr(astbuilder, fstr,
+        w_u, expr = fstring_find_literal_and_expr(astbuilder, fstr,
                                                       atom_node, rec)
 
         # add the literal part
-        f_constant_string(astbuilder, joined_pieces, literal, atom_node)
+        f_constant_string(astbuilder, joined_pieces, w_u, atom_node)
 
         if expr is None:
             break         # We're done with this f-string.

From pypy.commits at gmail.com  Thu Aug  9 16:30:47 2018
From: pypy.commits at gmail.com (mattip)
Date: Thu, 09 Aug 2018 13:30:47 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: start to replace
 s.encode('utf8') with calls to unicode_encode_utf_8
Message-ID: <5b6ca477.1c69fb81.6b4f4.d8c1@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94986:3125af5a0967
Date: 2018-08-09 13:29 -0700
http://bitbucket.org/pypy/pypy/changeset/3125af5a0967/

Log:	start to replace s.encode('utf8') with calls to unicode_encode_utf_8

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -6,10 +6,10 @@
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import enforceargs
 from rpython.rlib.rstring import StringBuilder
+from rpython.rlib.runicode import unicode_encode_utf_8
 
 from pypy.interpreter.error import OperationError, oefmt
 
-
 class Arguments(object):
     """
     Collects the arguments of a function call.
@@ -603,7 +603,8 @@
     def getmsg(self):
         if self.num_kwds == 1:
             if isinstance(self.kwd_name, unicode):
-                uname = self.kwd_name.encode('utf8')
+                uname = unicode_encode_utf_8(self.kwd_name, len(self.kwd_name),
+                        'strict', allow_surroagates=False)
             else:
                 uname = self.kwd_name
             msg = "got an unexpected keyword argument '%s'" % uname

From pypy.commits at gmail.com  Thu Aug  9 16:30:49 2018
From: pypy.commits at gmail.com (mattip)
Date: Thu, 09 Aug 2018 13:30:49 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: fix test
Message-ID: <5b6ca479.1c69fb81.2371f.1688@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94987:d7d67afa0ca8
Date: 2018-08-09 13:29 -0700
http://bitbucket.org/pypy/pypy/changeset/d7d67afa0ca8/

Log:	fix test

diff --git a/pypy/module/_codecs/test/test_locale.py b/pypy/module/_codecs/test/test_locale.py
--- a/pypy/module/_codecs/test/test_locale.py
+++ b/pypy/module/_codecs/test/test_locale.py
@@ -49,7 +49,7 @@
         utf8_encoder = self.getencoder('utf-8')
         encode_error_handler = self.getstate().encode_error_handler
         for val in u'foo\udc80bar', u'\udcff\U0001320C':
-            expected = utf8_encoder(val, 'surrogateescape',
+            expected = utf8_encoder(val.encode('utf8'), 'surrogateescape',
                                     encode_error_handler)
             assert locale_encoder(val).encode('utf8') == expected
 

From pypy.commits at gmail.com  Thu Aug  9 16:38:01 2018
From: pypy.commits at gmail.com (arigo)
Date: Thu, 09 Aug 2018 13:38:01 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2839
Message-ID: <5b6ca629.1c69fb81.8370.a6d8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94988:dc18c3e11193
Date: 2018-08-09 22:37 +0200
http://bitbucket.org/pypy/pypy/changeset/dc18c3e11193/

Log:	Issue #2839

	max(list-of-int) was much slower when run from non-jitted code.
	Fixed.

diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -209,8 +209,12 @@
 
 @specialize.arg(2)
 def min_max(space, args, implementation_of):
-    if not jit.we_are_jitted() or len(args.arguments_w) != 1 and \
-            jit.loop_unrolling_heuristic(args.arguments_w, len(args.arguments_w)):
+    # the 'normal' version includes a JIT merge point, which will make a
+    # new loop (from the interpreter or from another JIT loop).  If we
+    # give exactly two arguments to the call to max(), or a JIT virtual
+    # list of arguments, then we pick the 'unroll' version with no JIT
+    # merge point.
+    if jit.isvirtual(args.arguments_w) or len(args.arguments_w) == 2:
         return min_max_unroll(space, args, implementation_of)
     else:
         return min_max_normal(space, args, implementation_of)

From pypy.commits at gmail.com  Sat Aug 11 13:29:11 2018
From: pypy.commits at gmail.com (arigo)
Date: Sat, 11 Aug 2018 10:29:11 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Another test that used to crash before
 revision 179c172169f1
Message-ID: <5b6f1ce7.1c69fb81.3ef6.2112@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94989:476f93d18c1d
Date: 2018-08-11 19:28 +0200
http://bitbucket.org/pypy/pypy/changeset/476f93d18c1d/

Log:	Another test that used to crash before revision 179c172169f1

diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py
--- a/pypy/interpreter/test/test_syntax.py
+++ b/pypy/interpreter/test/test_syntax.py
@@ -701,6 +701,15 @@
             pass
         """
 
+    def test_bug_annotation_inside_nested_function(self):
+        """
+        # this used to crash
+        def f1():
+            def f2(*args: int):
+                pass
+        f1()
+        """
+
 class AppTestSyntaxError:
 
     def test_tokenizer_error_location(self):

From pypy.commits at gmail.com  Sat Aug 11 13:43:44 2018
From: pypy.commits at gmail.com (arigo)
Date: Sat, 11 Aug 2018 10:43:44 -0700 (PDT)
Subject: [pypy-commit] pypy default: Remove dead import;
 avoid using we_are_jitted() too much
Message-ID: <5b6f2050.1c69fb81.9e48c.cd37@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94990:29eab8b5cf20
Date: 2018-08-11 19:43 +0200
http://bitbucket.org/pypy/pypy/changeset/29eab8b5cf20/

Log:	Remove dead import; avoid using we_are_jitted() too much

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -1,7 +1,6 @@
 """The builtin str implementation"""
 
 from rpython.rlib import jit
-from rpython.rlib.jit import we_are_jitted
 from rpython.rlib.objectmodel import (
     compute_hash, compute_unique_id, import_from_mixin)
 from rpython.rlib.buffer import StringBuffer

From pypy.commits at gmail.com  Sun Aug 12 02:40:25 2018
From: pypy.commits at gmail.com (mattip)
Date: Sat, 11 Aug 2018 23:40:25 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: cleanups: typos,
 test adjustements, encode/decode disambiguation
Message-ID: <5b6fd659.1c69fb81.6f67c.b0e4@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94992:fcc25b6ffd38
Date: 2018-08-11 13:01 -0700
http://bitbucket.org/pypy/pypy/changeset/fcc25b6ffd38/

Log:	cleanups: typos, test adjustements, encode/decode disambiguation

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -604,7 +604,7 @@
         if self.num_kwds == 1:
             if isinstance(self.kwd_name, unicode):
                 uname = unicode_encode_utf_8(self.kwd_name, len(self.kwd_name),
-                        'strict', allow_surroagates=False)
+                        'strict', allow_surrogates=False)
             else:
                 uname = self.kwd_name
             msg = "got an unexpected keyword argument '%s'" % uname
diff --git a/pypy/interpreter/test/test_error.py b/pypy/interpreter/test/test_error.py
--- a/pypy/interpreter/test/test_error.py
+++ b/pypy/interpreter/test/test_error.py
@@ -19,7 +19,7 @@
     assert strings == ("abc ", " def ", "")
     assert issubclass(cls, OperationError)
     inst = cls("w_type", strings, "hello", 42)
-    assert inst._compute_value(space) == "abc hello def 42"
+    assert inst._compute_value(space) == ("abc hello def 42", 16)
     cls2, strings2 = get_operrcls2('a %s b %d c')
     assert cls2 is cls     # caching
     assert strings2 == ("a ", " b ", " c")
@@ -30,8 +30,7 @@
     assert operr.w_type == "w_type"
     assert operr._w_value is None
     val = operr._compute_value(space)
-    assert val == u"abc foo def 42"
-    assert isinstance(val, unicode)
+    assert val == ("abc foo def 42", 14)
     operr2 = oefmt("w_type2", "a %s b %d c", "bar", 43)
     assert operr2.__class__ is operr.__class__
     operr3 = oefmt("w_type2", "a %s b %s c", "bar", "4b")
@@ -49,49 +48,49 @@
     operr = oefmt(space.w_AttributeError,
                   "'%T' object has no attribute '%s'",
                   space.wrap('foo'), 'foo')
-    assert operr._compute_value(space) == "'str' object has no attribute 'foo'"
+    assert operr._compute_value(space) == ("'str' object has no attribute 'foo'", 35)
     operr = oefmt("w_type",
                   "'%T' object has no attribute '%s'",
                   space.wrap('foo'), 'foo')
-    assert operr._compute_value(space) == "'str' object has no attribute 'foo'"
+    assert operr._compute_value(space) == ("'str' object has no attribute 'foo'", 35)
 
 def test_oefmt_N(space):
     operr = oefmt(space.w_AttributeError,
                   "'%N' object has no attribute '%s'",
                   space.type(space.wrap('foo')), 'foo')
-    assert operr._compute_value(space) == "'str' object has no attribute 'foo'"
+    assert operr._compute_value(space) == ("'str' object has no attribute 'foo'", 35)
     operr = oefmt("w_type",
                   "'%N' object has no attribute '%s'",
                   space.type(space.wrap('foo')), 'foo')
-    assert operr._compute_value(space) == "'str' object has no attribute 'foo'"
+    assert operr._compute_value(space) == ("'str' object has no attribute 'foo'", 35)
     operr = oefmt(space.w_AttributeError,
                   "'%N' object has no attribute '%s'",
                   space.wrap('foo'), 'foo')
-    assert operr._compute_value(space) == "'?' object has no attribute 'foo'"
+    assert operr._compute_value(space) == ("'?' object has no attribute 'foo'", 33)
     operr = oefmt("w_type",
                   "'%N' object has no attribute '%s'",
                   space.wrap('foo'), 'foo')
-    assert operr._compute_value(space) == "'?' object has no attribute 'foo'"
+    assert operr._compute_value(space) == ("'?' object has no attribute 'foo'", 33)
 
 def test_oefmt_R(space):
     operr = oefmt(space.w_ValueError,
                   "illegal newline value: %R", space.wrap('foo'))
-    assert operr._compute_value(space) == "illegal newline value: 'foo'"
+    assert operr._compute_value(space) == ("illegal newline value: 'foo'", 28)
     operr = oefmt(space.w_ValueError, "illegal newline value: %R",
                   space.wrap("'PyLadies'"))
-    expected = "illegal newline value: \"'PyLadies'\""
+    expected = ("illegal newline value: \"'PyLadies'\"", 35)
     assert operr._compute_value(space) == expected
 
 def test_oefmt_unicode(space):
     operr = oefmt("w_type", "abc %s", u"&#224;&#232;&#236;&#242;&#249;")
     val = operr._compute_value(space)
-    assert val == u"abc &#224;&#232;&#236;&#242;&#249;"
+    assert val == (u"abc &#224;&#232;&#236;&#242;&#249;".encode('utf8'), 9)
 
 def test_oefmt_utf8(space):
     arg = u"&#224;&#232;&#236;&#242;&#249;".encode('utf-8')
     operr = oefmt("w_type", "abc %8", arg)
     val = operr._compute_value(space)
-    assert val == u"abc &#224;&#232;&#236;&#242;&#249;"
+    assert val == (u"abc &#224;&#232;&#236;&#242;&#249;".encode('utf8'), 9)
     #
     # if the arg is a byte string and we specify '%s', then we
     # also get utf-8 encoding.  This should be the common case
@@ -99,7 +98,7 @@
     # sources of PyPy.
     operr = oefmt("w_type", "abc %s", arg)
     val = operr._compute_value(space)
-    assert val == u"abc &#224;&#232;&#236;&#242;&#249;"
+    assert val == (u"abc &#224;&#232;&#236;&#242;&#249;".encode('utf8'), 9)
     #
     # if the byte string is not valid utf-8, then don't crash
     arg = '\xe9'
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -503,14 +503,21 @@
     message = "%s with '%s' codec failed" % (action, encoding)
     return operr.try_set_from_cause(space, message)
 
-def _call_codec(space, w_decoder, w_obj, action, encoding, errors):
+def _call_codec(space, w_coder, w_obj, action, encoding, errors):
     try:
-        w_res = space.call_function(w_decoder, w_obj, space.newtext(errors))
+        w_res = space.call_function(w_coder, w_obj, space.newtext(errors))
     except OperationError as operr:
         raise _wrap_codec_error(space, operr, action, encoding)
     if (not space.isinstance_w(w_res, space.w_tuple) or space.len_w(w_res) != 2):
-        raise oefmt(space.w_TypeError,
+        if action[:2] == 'en':
+            raise oefmt(space.w_TypeError,
                     "encoder must return a tuple (object, integer)")
+        elif action[:2] == 'de':
+            raise oefmt(space.w_TypeError,
+                    "decoder must return a tuple (object, integer)")
+        else:
+            raise oefmt(space.w_TypeError,
+                    "%s must return a tuple (object, integer)", action)
     return space.getitem(w_res, space.newint(0))
 
 @unwrap_spec(errors='text')
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -1226,14 +1226,20 @@
     return w_retval
 
 
-def decode_object(space, w_obj, encoding, errors):
+def decode_object(space, w_obj, encoding, errors='strict'):
+    assert errors is not None
     if encoding is None:
         encoding = getdefaultencoding(space)
-    if errors is None or errors == 'strict' or errors == 'surrogateescape':
+    if errors == 'surrogateescape':
+        s = space.charbuf_w(w_obj)
+        s, lgt, pos = unicodehelper.str_decode_utf8(s, errors, True,
+                    unicodehelper.decode_surrogateescape, True)
+        return space.newutf8(s, pos)
+    elif errors == 'strict':
         if encoding == 'ascii':
             s = space.charbuf_w(w_obj)
             unicodehelper.check_ascii_or_raise(space, s)
-            return space.newutf8(s, len(s))
+            return space.newtext(s, len(s))
         if encoding == 'utf-8' or encoding == 'utf8':
             s = space.charbuf_w(w_obj)
             lgt = unicodehelper.check_utf8_or_raise(space, s)

From pypy.commits at gmail.com  Sun Aug 12 02:40:27 2018
From: pypy.commits at gmail.com (mattip)
Date: Sat, 11 Aug 2018 23:40:27 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: change for utf8 not unicode
Message-ID: <5b6fd65b.1c69fb81.4b2f1.13bd@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94993:dacb894c61e0
Date: 2018-08-11 23:35 -0700
http://bitbucket.org/pypy/pypy/changeset/dacb894c61e0/

Log:	change for utf8 not unicode

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -263,13 +263,8 @@
             msg = "ordinal not in range(256)"
             res_8, newindex = errorhandler(
                 errors, 'latin1', msg, s, startindex, index)
-            for cp in rutf8.Utf8StringIterator(res_8):
-                if cp > 0xFF:
-                    errorhandler("strict", 'latin1', msg, s, startindex, index)
-                result.append(chr(cp))
-            if index != newindex:  # Should be uncommon
-                index = newindex
-                pos = rutf8._pos_at_index(s, newindex)
+            result.append(res_8)
+            pos = rutf8._pos_at_index(s, newindex)
     return result.build()
 
 def utf8_encode_ascii(s, errors, errorhandler):
@@ -296,13 +291,8 @@
             msg = "ordinal not in range(128)"
             res_8, newindex = errorhandler(
                 errors, 'ascii', msg, s, startindex, index)
-            for cp in rutf8.Utf8StringIterator(res_8):
-                if cp > 0x7F:
-                    errorhandler("strict", 'ascii', msg, s, startindex, index)
-                result.append(chr(cp))
-            if index != newindex:  # Should be uncommon
-                index = newindex
-                pos = rutf8._pos_at_index(s, newindex)
+            result.append(res_8)
+            pos = rutf8._pos_at_index(s, newindex)
     return result.build()
 
 if sys.platform == 'win32':

From pypy.commits at gmail.com  Sun Aug 12 02:40:29 2018
From: pypy.commits at gmail.com (mattip)
Date: Sat, 11 Aug 2018 23:40:29 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: add mising
 decode_surrogateescape, implement more of encode_error_handler
Message-ID: <5b6fd65d.1c69fb81.49f87.8622@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94994:215aee0a5be7
Date: 2018-08-11 23:36 -0700
http://bitbucket.org/pypy/pypy/changeset/215aee0a5be7/

Log:	add mising decode_surrogateescape, implement more of
	encode_error_handler

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -36,6 +36,26 @@
     ux = ['\ux' + hex(ord(x))[2:].upper() for x in s[startingpos:endingpos]]
     return ''.join(ux), endingpos
 
+def decode_surrogateescape(errors, encoding, msg, obj, start, end):
+    consumed = 0
+    replace = u''
+    while consumed < 4 and consumed < end - start:
+        c = ord(obj[start+consumed])
+        if c < 128:
+            # Refuse to escape ASCII bytes.
+            break
+        replace += unichr(0xdc00 + c)
+        consumed += 1
+    if not consumed:
+        # codec complained about ASCII byte.
+        raise OperationError(space.w_UnicodeDecodeError,
+                         space.newtuple([space.newtext(encoding),
+                                         space.newbytes(obj),
+                                         space.newint(start),
+                                         space.newint(end),
+                                         space.newtext(msg)]))
+    return replace.encode('utf8'), start + consumed
+
 @specialize.memo()
 def encode_error_handler(space):
     # Fast version of the "strict" errors handler.
@@ -227,12 +247,16 @@
     return res.build(), len(s), len(s)
 
 def utf8_encode_utf_8(s, errors, errorhandler, allow_surrogates=False):
-    # XXX completly implement this
     try:
         lgt = rutf8.check_utf8(s, allow_surrogates=allow_surrogates)
     except rutf8.CheckError as e:
-        s, lgt = errorhandler(errors, 'encoding',
+        # XXX change this to non-recursive
+        start = s[:e.pos]
+        ru, lgt = errorhandler(errors, 'utf8',
                     'surrogates not allowed', s, e.pos, e.pos + 1)
+        end = utf8_encode_utf_8(s[e.pos+3:], errors, errorhandler,
+                                allow_surrogates=allow_surrogates)
+        s = start + ru + end
     return s
 
 def utf8_encode_latin_1(s, errors, errorhandler):

From pypy.commits at gmail.com  Sun Aug 12 02:40:31 2018
From: pypy.commits at gmail.com (mattip)
Date: Sat, 11 Aug 2018 23:40:31 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: whoops
Message-ID: <5b6fd65f.1c69fb81.76e3f.6afa@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94995:e0e08fb400aa
Date: 2018-08-11 23:37 -0700
http://bitbucket.org/pypy/pypy/changeset/e0e08fb400aa/

Log:	whoops

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -970,7 +970,7 @@
         errors = 'strict'
     final = space.is_true(w_final)
     state = space.fromcache(CodecState)
-    result, lgt, u_len = unicodehelper.str_decode_raw_unicode_escape(
+    result, u_len, lgt = unicodehelper.str_decode_raw_unicode_escape(
         string, errors, final, state.decode_error_handler)
     return space.newtuple([space.newtext(result), space.newint(lgt)])
 

From pypy.commits at gmail.com  Sun Aug 12 02:40:33 2018
From: pypy.commits at gmail.com (mattip)
Date: Sat, 11 Aug 2018 23:40:33 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: change default values
Message-ID: <5b6fd661.1c69fb81.78439.73d8@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94996:2b95af3762f9
Date: 2018-08-11 23:38 -0700
http://bitbucket.org/pypy/pypy/changeset/2b95af3762f9/

Log:	change default values

diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -193,6 +193,8 @@
             _get_encoding_and_errors, decode_object)
         encoding, errors = _get_encoding_and_errors(space, w_encoding,
                                                     w_errors)
+        if errors is None:
+            errors = 'strict'
         return decode_object(space, self, encoding, errors)
 
     @unwrap_spec(tabsize=int)
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -515,7 +515,7 @@
     def descr_encode(self, space, w_encoding=None, w_errors=None):
         encoding, errors = _get_encoding_and_errors(space, w_encoding,
                                                     w_errors)
-        return encode_object(space, self, encoding, errors, allow_surrogates=True)
+        return encode_object(space, self, encoding, errors, allow_surrogates=False)
 
     @unwrap_spec(tabsize=int)
     def descr_expandtabs(self, space, tabsize=8):
@@ -670,7 +670,7 @@
 
     def descr_add(self, space, w_other):
         try:
-            w_other = self.convert_arg_to_w_unicode(space, w_other)
+            w_other = self.convert_arg_to_w_unicode(space, w_other, strict=True)
         except OperationError as e:
             if e.match(space, space.w_TypeError):
                 return space.w_NotImplemented
@@ -1191,13 +1191,19 @@
     utf8 = space.utf8_w(w_object)
     # TODO: refactor unnatrual use of error hanlders here,
     # we should make a single pass over the utf8 str
+    from pypy.module._codecs.interp_codecs import encode_text, CodecState
     if not allow_surrogates:
+        if errors is None:
+            errors = 'strict'
         pos = rutf8.surrogate_in_utf8(utf8)
         if pos >= 0:
-            eh = unicodehelper.encode_error_handler(space)
-            eh(None, "utf8", "surrogates not allowed", utf8,
+            state = space.fromcache(CodecState)
+            eh = state.encode_error_handler
+            start = utf8[:pos]
+            ru, pos = eh(errors, "utf8", "surrogates not allowed", utf8,
                 pos, pos + 1)
-            assert False, "always raises"
+            end = utf8[pos+1:]
+            utf8 = start + ru + end
     if errors is None or errors == 'strict':
         if encoding is None or encoding == 'utf-8':
             #if rutf8.has_surrogates(utf8):
@@ -1213,7 +1219,6 @@
                 assert False, "always raises"
             return space.newbytes(utf8)
 
-    from pypy.module._codecs.interp_codecs import encode_text
     if encoding is None:
         encoding = space.sys.defaultencoding
     w_retval = encode_text(space, w_object, encoding, errors)
@@ -1228,8 +1233,7 @@
 
 def decode_object(space, w_obj, encoding, errors='strict'):
     assert errors is not None
-    if encoding is None:
-        encoding = getdefaultencoding(space)
+    assert encoding is not None
     if errors == 'surrogateescape':
         s = space.charbuf_w(w_obj)
         s, lgt, pos = unicodehelper.str_decode_utf8(s, errors, True,
@@ -1256,6 +1260,10 @@
 
 
 def unicode_from_encoded_object(space, w_obj, encoding, errors):
+    if errors is None:
+        errors = 'strict'
+    if encoding is None:
+        encoding = getdefaultencoding(space)
     w_retval = decode_object(space, w_obj, encoding, errors)
     if not space.isinstance_w(w_retval, space.w_unicode):
         raise oefmt(space.w_TypeError,

From pypy.commits at gmail.com  Sun Aug 12 02:40:35 2018
From: pypy.commits at gmail.com (mattip)
Date: Sat, 11 Aug 2018 23:40:35 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: bytes is already utf8,
 no need to decode
Message-ID: <5b6fd663.1c69fb81.f4984.d1b4@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94997:2b6c2810e8bc
Date: 2018-08-11 23:38 -0700
http://bitbucket.org/pypy/pypy/changeset/2b6c2810e8bc/

Log:	bytes is already utf8, no need to decode

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -423,9 +423,7 @@
                     "unicode object expected, received bytes instead")
 
     def utf8_w(self, space):
-        # Use the default encoding.                                             
-        encoding = getdefaultencoding(space)
-        return space.utf8_w(decode_object(space, self, encoding, None))
+        return self._value
 
     def buffer_w(self, space, flags):
         space.check_buf_flags(flags, True)

From pypy.commits at gmail.com  Sun Aug 12 14:10:45 2018
From: pypy.commits at gmail.com (arigo)
Date: Sun, 12 Aug 2018 11:10:45 -0700 (PDT)
Subject: [pypy-commit] cffi default: Refactor again overview.rst,
 including all cases in the order that seems
Message-ID: <5b707825.1c69fb81.890d2.16d5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3138:549cf1a22e97
Date: 2018-08-12 20:10 +0200
http://bitbucket.org/cffi/cffi/changeset/549cf1a22e97/

Log:	Refactor again overview.rst, including all cases in the order that
	seems to make the most sense to me

diff --git a/doc/source/overview.rst b/doc/source/overview.rst
--- a/doc/source/overview.rst
+++ b/doc/source/overview.rst
@@ -7,7 +7,7 @@
 
 This document starts, in the first section, with a simple working
 example of using CFFI to call a C function from Python. CFFI is
-flexible and covers several use cases presented in the second
+flexible and covers several other use cases presented in the second
 section. Then, the next section shows how to export Python functions
 to a Python interpreter embedded in a C or C++ application. The last
 two sections delve deeper in the CFFI library.
@@ -20,96 +20,65 @@
 .. _real-example:
 
 
-Example: calling a C function from Python
------------------------------------------
+Main mode of usage
+------------------
 
-This example is about the use case when the library sources are
-available, the next section shows how use a compiled, installed
-library.
+The main way to use CFFI is as an interface to some already-compiled
+library which is provided by other means.  Imagine that you have a
+system-installed library called ``piapprox.dll`` (Windows) or
+``libpiapprox.so`` (Linux and others) or ``libpiapprox.dylib`` (OS X),
+containing a function ``float pi_approx(int n);`` that computes some
+approximation of pi given a number of iterations.  You want to call
+this function from Python.
 
-1. Make sure the sources of the library defining the useful C function
-   is available. For this example, create the file ``pi.c`` and ``pi.h``:
+Create the file ``piapprox_build.py``:
 
-   .. code-block:: C
-
-      /* filename: pi.c*/
-      # include <stdlib.h>
-      # include <math.h>
-       
-      /* Returns a very crude approximation of Pi
-         given a int: a number of iteration */
-      float pi_approx(int n){
-      
-        double i,x,y,sum=0;
-      
-        for(i=0;i<n;i++){
-      
-          x=rand();
-          y=rand();
-      
-          if (sqrt(x*x+y*y) < sqrt((double)RAND_MAX*RAND_MAX))
-            sum++; }
-      
-        return 4*(float)sum/(float)n; }
-
-   .. code-block:: C
-
-      /* filename: pi.h*/
-      float pi_approx(int n);
-      
-2. Create a script named ``pi_extension_build.py``, building
-   the C extension:
-   
-   .. code-block:: python
+.. code-block:: python
 
       from cffi import FFI
       ffibuilder = FFI()
-      
-      # cdef() expects a string listing the C types, functions and 
-      # globals needed from Python. The string is in the C syntax,
-      # which saves us from learning a Python descriptive dialect.
-      ffibuilder.cdef("float pi_approx(int n);")
-   
-      ffibuilder.set_source(       
-          "_pi",  # name of the output C extension
-      	   '# include "pi.h"',
-      	   sources=['pi.c'],
-      	   libraries=['m'])
-   
+
+      # cdef() expects a string listing the C types, functions and
+      # globals needed from Python. The string follows the C syntax.
+      ffibuilder.cdef("""
+          float pi_approx(int n);
+      """)
+
+      # This describes the extension module "_pi_cffi" to produce.
+      ffibuilder.set_source("_pi_cffi",
+      """
+      	   #include "pi.h"   // the C header of the library
+      """,
+      	   libraries=['piapprox'])   # library name, for the linker
+
       if __name__ == "__main__":
           ffibuilder.compile(verbose=True)
 
-       
-3. Build the extension:
-   
-   .. code-block:: shell
+Execute this script.  If everything is OK, it should produce
+``_pi_cffi.c``, and then invoke the compiler on it.  The produced
+``_pi_cffi.c`` contains a copy of the string given in ``set_source()``,
+in this example the ``#include "pi.h"``; then it contains some glue code
+for all the functions declared in the ``cdef()`` above.
 
-      python pi_extension_build.py
+At runtime, you use the extension module like this:
 
-   Observe, in the working directory, the generated output files:
-   ``_pi.c``, ``_pi.o`` and the C extension ``_pi.so``.
+.. code-block:: python
 
+    from _pi_cffi import ffi, lib
+    print(lib.pi_approx(5000))
 
-4. Call the C function from Python:
+That's all!  In the rest of this page, we describe some more advanced
+examples and other CFFI modes.  In particular, there is a complete
+example `if you don't have an already-installed C library to call`_.
 
-   .. code-block:: python
-   
-       from _pi.lib import pi_approx
-   
-       approx = pi_approx(10)
-       assert str(pi_approximation).startswith("3.")
-   
-       approx = pi_approx(10000)
-       assert str(approx).startswith("3.1")  
-
-For more information, see the ``cdef()`` and ``set_source()`` methods
-of the ``FFI`` class covered in `Preparing and Distributing modules`__.
+For more information about the ``cdef()`` and ``set_source()`` methods
+of the ``FFI`` class, see `Preparing and Distributing modules`__.
 
 .. __: cdef.html
 
-
-A common alternative for the step 3. and running the build script is
-to write a ``setup.py`` Setuptools distribution:
+When your example works, a common alternative to running the build
+script manually is to have it run as part of a ``setup.py``.  Here is
+an example using the Setuptools distribution:
 
 .. code-block:: python
 
@@ -118,13 +87,10 @@
     setup(
         ...
         setup_requires=["cffi>=1.0.0"],
-        cffi_modules=["pi_extension_build:ffibuilder"],
+        cffi_modules=["piapprox_build:ffibuilder"], # "filename:global"
         install_requires=["cffi>=1.0.0"],
     )
 
-``cffi_modules`` is a list of ``<extension builder script>:<FFI
-instance>`` describing the modules to build.
-
 
 Other CFFI modes
 ----------------
@@ -133,8 +99,8 @@
 each with "in-line" or "out-of-line" preparation (or compilation).
 
 The **ABI mode** accesses libraries at the binary level, whereas the
-faster **API mode** accesses them with a C compiler.  This is described in
-detail below__.
+faster **API mode** accesses them with a C compiler.  We explain the
+difference in more details below__.
 
 .. __: `abi-versus-api`_
 
@@ -176,9 +142,7 @@
 cdef().*
 
 If using a C compiler to install your module is an option, it is highly
-recommended to use the API mode described in the next paragraph.  (It is
-also faster.)
-
+recommended to use the API mode instead.  (It is also faster.)
 
 
 Struct/Array Example (minimal, in-line)
@@ -217,7 +181,7 @@
 *This example does not call any C compiler.*
 
 This example also admits an out-of-line equivalent.  It is similar to
-the first example `Example: calling a C function from Python`_ above,
+the first example `Main mode of usage`_ above,
 but passing ``None`` as the second argument to
 ``ffibuilder.set_source()``.  Then in the main program you write
 ``from _simple_example import ffi`` and then the same content as the
@@ -225,6 +189,173 @@
 ffi.new("pixel_t[]", 800*600)``.
 
 
+API Mode, calling the C standard library
+++++++++++++++++++++++++++++++++++++++++
+
+.. code-block:: python
+
+    # file "example_build.py"
+
+    # Note: we instantiate the same 'cffi.FFI' class as in the previous
+    # example, but call the result 'ffibuilder' now instead of 'ffi';
+    # this is to avoid confusion with the other 'ffi' object you get below
+
+    from cffi import FFI
+    ffibuilder = FFI()
+
+    ffibuilder.set_source("_example",
+       r""" // passed to the real C compiler,
+            // contains implementation of things declared in cdef()
+            #include <sys/types.h>
+            #include <pwd.h>
+
+            // We can also define custom wrappers or other functions
+            // here (this is an example only):
+            static struct passwd *get_pw_for_root(void) {
+                return getpwuid(0);
+            }
+        """,
+        libraries=[])   # or a list of libraries to link with
+        # (more arguments like setup.py's Extension class:
+        # include_dirs=[..], extra_objects=[..], and so on)
+
+    ffibuilder.cdef("""
+        // declarations that are shared between Python and C
+        struct passwd {
+            char *pw_name;
+            ...;     // literally dot-dot-dot
+        };
+        struct passwd *getpwuid(int uid);     // defined in <pwd.h>
+        struct passwd *get_pw_for_root(void); // defined in set_source()
+    """)
+
+    if __name__ == "__main__":
+        ffibuilder.compile(verbose=True)
+
+You need to run the ``example_build.py`` script once to generate
+"source code" into the file ``_example.c`` and compile this to a
+regular C extension module.  (CFFI selects either Python or C for the
+module to generate based on whether the second argument to
+``set_source()`` is ``None`` or not.)
+
+*You need a C compiler for this single step.  It produces a file called
+e.g. _example.so or _example.pyd.  If needed, it can be distributed in
+precompiled form like any other extension module.*
+
+Then, in your main program, you use:
+
+.. code-block:: python
+
+    from _example import ffi, lib
+
+    p = lib.getpwuid(0)
+    assert ffi.string(p.pw_name) == b'root'
+    p = lib.get_pw_for_root()
+    assert ffi.string(p.pw_name) == b'root'
+
+Note that this works independently of the exact C layout of ``struct
+passwd`` (it is "API level", as opposed to "ABI level").  It requires
+a C compiler in order to run ``example_build.py``, but it is much more
+portable than trying to get the details of the fields of ``struct
+passwd`` exactly right.  Similarly, in the ``cdef()`` we declared
+``getpwuid()`` as taking an ``int`` argument; on some platforms this
+might be slightly incorrect---but it does not matter.
+
+Note also that at runtime, the API mode is faster than the ABI mode.
+
+To integrate it inside a ``setup.py`` distribution with Setuptools:
+
+.. code-block:: python
+
+    from setuptools import setup
+
+    setup(
+        ...
+        setup_requires=["cffi>=1.0.0"],
+        cffi_modules=["example_build.py:ffibuilder"],
+        install_requires=["cffi>=1.0.0"],
+    )
+
+
+.. _`if you don't have an already-installed C library to call`:
+
+API Mode, calling C sources instead of a compiled library
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+If you want to call some library that is not precompiled, but for which
+you have C sources, then the easiest solution is to make a single
+extension module that is compiled from both the C sources of this
+library, and the additional CFFI wrappers.  For example, say you start
+with the files ``pi.c`` and ``pi.h``:
+
+   .. code-block:: C
+
+      /* filename: pi.c*/
+      # include <stdlib.h>
+      # include <math.h>
+       
+      /* Returns a very crude approximation of Pi
+         given a int: a number of iteration */
+      float pi_approx(int n){
+      
+        double i,x,y,sum=0;
+      
+        for(i=0;i<n;i++){
+      
+          x=rand();
+          y=rand();
+      
+          if (sqrt(x*x+y*y) < sqrt((double)RAND_MAX*RAND_MAX))
+            sum++; }
+      
+        return 4*(float)sum/(float)n; }
+
+   .. code-block:: C
+
+      /* filename: pi.h*/
+      float pi_approx(int n);
+      
+Create a script named ``pi_extension_build.py``, building
+the C extension:
+
+   .. code-block:: python
+
+      from cffi import FFI
+      ffibuilder = FFI()
+      
+      ffibuilder.cdef("float pi_approx(int n);")
+   
+      ffibuilder.set_source("_pi",  # name of the output C extension
+      """
+          #include "pi.h"',
+      """,
+          sources=['pi.c'],   # includes pi.c as additional sources
+          libraries=['m'])    # on Unix, link with the math library
+   
+      if __name__ == "__main__":
+          ffibuilder.compile(verbose=True)
+
+Build the extension:
+   
+   .. code-block:: shell
+
+      python pi_extension_build.py
+
+Observe, in the working directory, the generated output files:
+``_pi.c``, ``_pi.o`` and the compiled C extension (called ``_pi.so`` on
+Linux for example).  It can be called from Python:
+
+   .. code-block:: python
+   
+       from _pi.lib import pi_approx
+   
+       approx = pi_approx(10)
+       assert str(pi_approximation).startswith("3.")
+   
+       approx = pi_approx(10000)
+       assert str(approx).startswith("3.1")  
+
+
 .. _performance:
 
 Purely for performance (API level, out-of-line)
@@ -347,6 +478,16 @@
     )
 
 
+In-line, API level
+++++++++++++++++++
+
+The "API level + in-line" mode combination exists but is long
+deprecated.  It used to be done with ``lib = ffi.verify("C header")``.
+The out-of-line variant with ``set_source("modname", "C header")`` is
+preferred.  It avoids a number of problems when the project grows in
+size.
+
+
 .. _embedding:
 
 Embedding
@@ -495,8 +636,3 @@
 should include or not.
 
 .. __: real-example_
-
-Note that the "API level + in-line" mode combination exists but is long
-deprecated.  It used to be done with ``lib = ffi.verify("C header")``.
-The out-of-line variant with ``set_source("modname", "C header")`` is
-preferred.

From pypy.commits at gmail.com  Sun Aug 12 17:16:11 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 12 Aug 2018 14:16:11 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: specify errors,
 disallow space.text_w(non-unicode) but allow W_Bytes.text_w
Message-ID: <5b70a39b.1c69fb81.6c542.6900@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94998:e5867f1518c9
Date: 2018-08-12 14:13 -0700
http://bitbucket.org/pypy/pypy/changeset/e5867f1518c9/

Log:	specify errors, disallow space.text_w(non-unicode) but allow
	W_Bytes.text_w

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1618,6 +1618,8 @@
             an utf-8 encoded rpython string.
         """
         assert w_obj is not None
+        if not self.isinstance_w(w_obj, self.w_unicode):
+            w_obj._typed_unwrap_error(self, "unicode")
         return w_obj.text_w(self)
 
     @not_rpython    # tests only; should be replaced with bytes_w or text_w
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -422,6 +422,9 @@
         raise oefmt(space.w_TypeError,
                     "unicode object expected, received bytes instead")
 
+    def text_w(self, space):
+        return self._value
+
     def utf8_w(self, space):
         return self._value
 
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -195,6 +195,8 @@
                                                     w_errors)
         if errors is None:
             errors = 'strict'
+        if encoding is None:
+            encoding = 'utf8'
         return decode_object(space, self, encoding, errors)
 
     @unwrap_spec(tabsize=int)
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -670,7 +670,7 @@
 
     def descr_add(self, space, w_other):
         try:
-            w_other = self.convert_arg_to_w_unicode(space, w_other, strict=True)
+            w_other = self.convert_arg_to_w_unicode(space, w_other, strict='__add__')
         except OperationError as e:
             if e.match(space, space.w_TypeError):
                 return space.w_NotImplemented
@@ -1285,7 +1285,7 @@
     # repr is guaranteed to be unicode
     w_repr = space.repr(w_obj)
     w_encoded = encode_object(space, w_repr, 'ascii', 'backslashreplace')
-    return decode_object(space, w_encoded, 'ascii', None)
+    return decode_object(space, w_encoded, 'ascii', 'strict')
 
 def unicode_from_string(space, w_bytes):
     # this is a performance and bootstrapping hack

From pypy.commits at gmail.com  Sun Aug 12 17:16:13 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 12 Aug 2018 14:16:13 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: do not special-case
 surrogateescape
Message-ID: <5b70a39d.1c69fb81.21873.a602@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r94999:51e9202ee008
Date: 2018-08-12 14:14 -0700
http://bitbucket.org/pypy/pypy/changeset/51e9202ee008/

Log:	do not special-case surrogateescape

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -27,8 +27,6 @@
                                              space.newint(startingpos),
                                              space.newint(endingpos),
                                              space.newtext(msg)]))
-        # make annotator happy
-        return '', 0
     return raise_unicode_exception_decode
 
 def decode_never_raise(errors, encoding, msg, s, startingpos, endingpos):
@@ -36,26 +34,6 @@
     ux = ['\ux' + hex(ord(x))[2:].upper() for x in s[startingpos:endingpos]]
     return ''.join(ux), endingpos
 
-def decode_surrogateescape(errors, encoding, msg, obj, start, end):
-    consumed = 0
-    replace = u''
-    while consumed < 4 and consumed < end - start:
-        c = ord(obj[start+consumed])
-        if c < 128:
-            # Refuse to escape ASCII bytes.
-            break
-        replace += unichr(0xdc00 + c)
-        consumed += 1
-    if not consumed:
-        # codec complained about ASCII byte.
-        raise OperationError(space.w_UnicodeDecodeError,
-                         space.newtuple([space.newtext(encoding),
-                                         space.newbytes(obj),
-                                         space.newint(start),
-                                         space.newint(end),
-                                         space.newtext(msg)]))
-    return replace.encode('utf8'), start + consumed
-
 @specialize.memo()
 def encode_error_handler(space):
     # Fast version of the "strict" errors handler.
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -1234,12 +1234,7 @@
 def decode_object(space, w_obj, encoding, errors='strict'):
     assert errors is not None
     assert encoding is not None
-    if errors == 'surrogateescape':
-        s = space.charbuf_w(w_obj)
-        s, lgt, pos = unicodehelper.str_decode_utf8(s, errors, True,
-                    unicodehelper.decode_surrogateescape, True)
-        return space.newutf8(s, pos)
-    elif errors == 'strict':
+    if errors == 'strict':
         if encoding == 'ascii':
             s = space.charbuf_w(w_obj)
             unicodehelper.check_ascii_or_raise(space, s)

From pypy.commits at gmail.com  Sun Aug 12 17:16:15 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 12 Aug 2018 14:16:15 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: fix translation of temporary
 code
Message-ID: <5b70a39f.1c69fb81.ef517.e09c@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95000:631c31e88913
Date: 2018-08-12 14:14 -0700
http://bitbucket.org/pypy/pypy/changeset/631c31e88913/

Log:	fix translation of temporary code

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -229,10 +229,12 @@
         lgt = rutf8.check_utf8(s, allow_surrogates=allow_surrogates)
     except rutf8.CheckError as e:
         # XXX change this to non-recursive
-        start = s[:e.pos]
+        pos = e.pos
+        assert pos >= 0
+        start = s[:pos]
         ru, lgt = errorhandler(errors, 'utf8',
-                    'surrogates not allowed', s, e.pos, e.pos + 1)
-        end = utf8_encode_utf_8(s[e.pos+3:], errors, errorhandler,
+                    'surrogates not allowed', s, pos, pos + 1)
+        end = utf8_encode_utf_8(s[pos+3:], errors, errorhandler,
                                 allow_surrogates=allow_surrogates)
         s = start + ru + end
     return s

From pypy.commits at gmail.com  Sun Aug 12 17:16:17 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 12 Aug 2018 14:16:17 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: whoops
Message-ID: <5b70a3a1.1c69fb81.d1055.1fa3@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95001:de720af40556
Date: 2018-08-12 14:14 -0700
http://bitbucket.org/pypy/pypy/changeset/de720af40556/

Log:	whoops

diff --git a/pypy/interpreter/astcompiler/fstring.py b/pypy/interpreter/astcompiler/fstring.py
--- a/pypy/interpreter/astcompiler/fstring.py
+++ b/pypy/interpreter/astcompiler/fstring.py
@@ -264,7 +264,7 @@
     if not fstr.raw_mode and '\\' in literal:
         literal = parsestring.decode_unicode_utf8(space, literal, 0,
                                                   len(literal))
-        literal, pos, lgt = unicodehelper.decode_unicode_escape(space, literal)
+        literal, lgt, pos = unicodehelper.decode_unicode_escape(space, literal)
     return space.newtext(literal, lgt)
 
 

From pypy.commits at gmail.com  Mon Aug 13 12:38:36 2018
From: pypy.commits at gmail.com (mattip)
Date: Mon, 13 Aug 2018 09:38:36 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix print statement
Message-ID: <5b71b40c.1c69fb81.2a069.4255@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r95002:d6a8f43fadd2
Date: 2018-08-13 09:37 -0700
http://bitbucket.org/pypy/pypy/changeset/d6a8f43fadd2/

Log:	fix print statement

diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py
--- a/pypy/module/cpyext/test/test_datetime.py
+++ b/pypy/module/cpyext/test/test_datetime.py
@@ -308,7 +308,7 @@
         # copied from datetime documentation
         class GMT1(tzinfo):
             def __del__(self):
-                print 'deleting GMT1'
+                print('deleting GMT1')
             def utcoffset(self, dt):
                 return timedelta(hours=1) + self.dst(dt)
             def dst(self, dt):

From pypy.commits at gmail.com  Mon Aug 13 12:41:15 2018
From: pypy.commits at gmail.com (mattip)
Date: Mon, 13 Aug 2018 09:41:15 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: fix lib_pypy tests
Message-ID: <5b71b4ab.1c69fb81.dcd88.772b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95003:82a40174a16f
Date: 2018-08-12 15:00 -0700
http://bitbucket.org/pypy/pypy/changeset/82a40174a16f/

Log:	fix lib_pypy tests

diff --git a/pypy/module/test_lib_pypy/test_md5_extra.py b/pypy/module/test_lib_pypy/test_md5_extra.py
--- a/pypy/module/test_lib_pypy/test_md5_extra.py
+++ b/pypy/module/test_lib_pypy/test_md5_extra.py
@@ -80,7 +80,7 @@
 
             space.call_method(w_m2c, 'update', space.newbytes(message))
             w_d2 = space.call_method(w_m2c, 'hexdigest')
-            d2 = space.str_w(w_d2)
+            d2 = space.text_w(w_d2)
 
             assert d1 == d2
 
diff --git a/pypy/module/test_lib_pypy/test_resource.py b/pypy/module/test_lib_pypy/test_resource.py
--- a/pypy/module/test_lib_pypy/test_resource.py
+++ b/pypy/module/test_lib_pypy/test_resource.py
@@ -1,14 +1,15 @@
 from __future__ import absolute_import
 import sys
+import pytest
 
 import os
 if os.name != 'posix':
-    skip('resource.h only available on unix')
+    pytest.skip('resource.h only available on unix')
 
 try:
     from lib_pypy import resource
 except (ImportError, SyntaxError) as e:
-    skip(str(e))
+    pytest.skip(str(e))
 
 
 def test_getrusage():
diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py
--- a/pypy/module/test_lib_pypy/test_sqlite3.py
+++ b/pypy/module/test_lib_pypy/test_sqlite3.py
@@ -274,14 +274,14 @@
     def test_null_character(self, con):
         if not hasattr(_sqlite3, '_ffi') and sys.version_info < (2, 7, 9):
             pytest.skip("_sqlite3 too old")
-        exc = raises(ValueError, con, "\0select 1")
+        exc = pytest.raises(ValueError, con, "\0select 1")
         assert str(exc.value) == "the query contains a null character"
-        exc = raises(ValueError, con, "select 1\0")
+        exc = pytest.raises(ValueError, con, "select 1\0")
         assert str(exc.value) == "the query contains a null character"
         cur = con.cursor()
-        exc = raises(ValueError, cur.execute, "\0select 2")
+        exc = pytest.raises(ValueError, cur.execute, "\0select 2")
         assert str(exc.value) == "the query contains a null character"
-        exc = raises(ValueError, cur.execute, "select 2\0")
+        exc = pytest.raises(ValueError, cur.execute, "select 2\0")
         assert str(exc.value) == "the query contains a null character"
 
     def test_close_in_del_ordering(self):

From pypy.commits at gmail.com  Mon Aug 13 12:41:18 2018
From: pypy.commits at gmail.com (mattip)
Date: Mon, 13 Aug 2018 09:41:18 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: merge py3.5 into branch
Message-ID: <5b71b4ae.1c69fb81.87af0.a88d@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95004:6ef4fd4b0165
Date: 2018-08-13 09:40 -0700
http://bitbucket.org/pypy/pypy/changeset/6ef4fd4b0165/

Log:	merge py3.5 into branch

diff too long, truncating to 2000 out of 5190 lines

diff --git a/lib-python/3/hashlib.py b/lib-python/3/hashlib.py
--- a/lib-python/3/hashlib.py
+++ b/lib-python/3/hashlib.py
@@ -134,9 +134,14 @@
     __get_hash = __get_openssl_constructor
     algorithms_available = algorithms_available.union(
             _hashlib.openssl_md_meth_names)
-except ImportError:
+except ImportError as e:
     new = __py_new
     __get_hash = __get_builtin_constructor
+    # added by PyPy
+    import warnings
+    warnings.warn("The _hashlib module is not available, falling back "
+                  "to a much slower implementation (%s)" % str(e),
+                  RuntimeWarning)
 
 try:
     # OpenSSL's PKCS5_PBKDF2_HMAC requires OpenSSL 1.0+ with HMAC and SHA
diff --git a/lib-python/3/types.py b/lib-python/3/types.py
--- a/lib-python/3/types.py
+++ b/lib-python/3/types.py
@@ -41,9 +41,19 @@
     FrameType = type(tb.tb_frame)
     tb = None; del tb
 
-# For Jython, the following two types are identical
+#
+# On CPython, FunctionType.__code__ is a 'getset_descriptor', but
+# FunctionType.__globals__ is a 'member_descriptor', just like app-level
+# slots.  On PyPy, all descriptors of built-in types are
+# 'getset_descriptor', but the app-level slots are 'member_descriptor'
+# as well.  (On Jython the situation might still be different.)
+#
+# Note that MemberDescriptorType was equal to GetSetDescriptorType in
+# PyPy <= 6.0.
+#
 GetSetDescriptorType = type(FunctionType.__code__)
-MemberDescriptorType = type(FunctionType.__globals__)
+class _C: __slots__ = 's'
+MemberDescriptorType = type(_C.s)
 
 del sys, _f, _g, _C, _c,                           # Not for export
 
diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py
--- a/lib_pypy/_winapi.py
+++ b/lib_pypy/_winapi.py
@@ -98,9 +98,11 @@
     def GetOverlappedResult(self, wait):
         transferred = _ffi.new('DWORD[1]', [0])
         res = _kernel32.GetOverlappedResult(self.handle, self.overlapped, transferred, wait != 0)
-        if not res:
-            res = GetLastError()
-        if res in (ERROR_SUCCESS, ERROR_MORE_DATA, ERROR_OPERATION_ABORTED):
+        if res:
+            err = ERROR_SUCCESS
+        else:
+            err = GetLastError()
+        if err in (ERROR_SUCCESS, ERROR_MORE_DATA, ERROR_OPERATION_ABORTED):
             self.completed = 1
             self.pending = 0
         elif res == ERROR_IO_INCOMPLETE:
@@ -133,7 +135,7 @@
         assert success == 0
         err = _kernel32.GetLastError()
         if err == ERROR_IO_PENDING:
-            overlapped[0].pending = 1
+            ov.pending = 1
         elif err == ERROR_PIPE_CONNECTED:
             _kernel32.SetEvent(ov.overlapped[0].hEvent)
         else:
diff --git a/lib_pypy/cffi/_cffi_errors.h b/lib_pypy/cffi/_cffi_errors.h
--- a/lib_pypy/cffi/_cffi_errors.h
+++ b/lib_pypy/cffi/_cffi_errors.h
@@ -50,7 +50,9 @@
         "import sys\n"
         "class FileLike:\n"
         "  def write(self, x):\n"
-        "    of.write(x)\n"
+        "    try:\n"
+        "      of.write(x)\n"
+        "    except: pass\n"
         "    self.buf += x\n"
         "fl = FileLike()\n"
         "fl.buf = ''\n"
diff --git a/lib_pypy/grp.py b/lib_pypy/grp.py
--- a/lib_pypy/grp.py
+++ b/lib_pypy/grp.py
@@ -5,8 +5,8 @@
 import os
 from _pwdgrp_cffi import ffi, lib
 import _structseq
-import thread
-_lock = thread.allocate_lock()
+import _thread
+_lock = _thread.allocate_lock()
 
 try: from __pypy__ import builtinify
 except ImportError: builtinify = lambda f: f
diff --git a/lib_pypy/pwd.py b/lib_pypy/pwd.py
--- a/lib_pypy/pwd.py
+++ b/lib_pypy/pwd.py
@@ -12,8 +12,8 @@
 
 from _pwdgrp_cffi import ffi, lib
 import _structseq
-import thread
-_lock = thread.allocate_lock()
+import _thread
+_lock = _thread.allocate_lock()
 
 try: from __pypy__ import builtinify
 except ImportError: builtinify = lambda f: f
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -43,14 +43,10 @@
     "_jitlog",
 ])
 
-from rpython.jit.backend import detect_cpu
-try:
-    if detect_cpu.autodetect().startswith('x86'):
-        if not sys.platform.startswith('openbsd'):
-            working_modules.add('_vmprof')
-            working_modules.add('faulthandler')
-except detect_cpu.ProcessorAutodetectError:
-    pass
+import rpython.rlib.rvmprof.cintf
+if rpython.rlib.rvmprof.cintf.IS_SUPPORTED:
+    working_modules.add('_vmprof')
+    working_modules.add('faulthandler')
 
 translation_modules = default_modules.copy()
 translation_modules.update([
@@ -323,3 +319,4 @@
     parser = to_optparse(config) #, useoptions=["translation.*"])
     option, args = parser.parse_args()
     print config
+    print working_modules
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -7,9 +7,13 @@
 
 .. branch: cppyy-packaging
 
-Upgrade to backend 1.1.0, improved handling of templated methods and
+Main items: vastly better template resolution and improved performance. In
+detail: upgrade to backend 1.4, improved handling of templated methods and
 functions (in particular automatic deduction of types), improved pythonization
-interface, and a range of compatibility fixes for Python3
+interface, range of compatibility fixes for Python3, free functions now take
+fast libffi path when possible, moves for strings (incl. from Python str),
+easier/faster handling of std::vector by numpy, improved and faster object
+identity preservation
 
 .. branch: socket_default_timeout_blockingness
 
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -367,8 +367,8 @@
                                        names)
         self._visit_arg_annotations(args.kwonlyargs, names)
         kwarg = args.kwarg
-        if args.kwarg:
-            self._visit_arg_annotation(args.kwarg.arg, args.kwarg.annotation,
+        if kwarg:
+            self._visit_arg_annotation(kwarg.arg, kwarg.annotation,
                                        names)
         self._visit_arg_annotation("return", returns, names)
         l = len(names)
diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -622,6 +622,10 @@
         assert isinstance(args, ast.arguments)
         if args.args:
             self._visit_arg_annotations(args.args)
+        if args.vararg:
+            self._visit_arg_annotation(args.vararg)
+        if args.kwarg:
+            self._visit_arg_annotation(args.kwarg)
         if args.kwonlyargs:
             self._visit_arg_annotations(args.kwonlyargs)
         if func.returns:
@@ -630,8 +634,11 @@
     def _visit_arg_annotations(self, args):
         for arg in args:
             assert isinstance(arg, ast.arg)
-            if arg.annotation:
-                arg.annotation.walkabout(self)
+            self._visit_arg_annotation(arg)
+
+    def _visit_arg_annotation(self, arg):
+        if arg.annotation:
+            arg.annotation.walkabout(self)
 
     def visit_Name(self, name):
         if name.ctx == ast.Load:
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -2076,7 +2076,7 @@
         else:
             skip_leading_underscores = False
         for name in all:
-            if skip_leading_underscores and name[0]=='_':
+            if skip_leading_underscores and name and name[0] == '_':
                 continue
             into_locals[name] = getattr(module, name)
 ''', filename=__file__)
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -470,7 +470,7 @@
 
     def test_cmd_co_name(self):
         child = self.spawn(['-c',
-                    'import sys; print sys._getframe(0).f_code.co_name'])
+                    'import sys; print(sys._getframe(0).f_code.co_name)'])
         child.expect('<module>')
 
     def test_ignore_python_inspect(self):
diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py
--- a/pypy/interpreter/test/test_executioncontext.py
+++ b/pypy/interpreter/test/test_executioncontext.py
@@ -149,7 +149,7 @@
         pass
         """)
         space.getexecutioncontext().setllprofile(None, None)
-        assert l == ['call', 'return', 'call', 'return']
+        assert l[-4:] == ['call', 'return', 'call', 'return']
 
     def test_llprofile_c_call(self):
         from pypy.interpreter.function import Function, Method
@@ -173,15 +173,15 @@
             return
             """ % snippet)
             space.getexecutioncontext().setllprofile(None, None)
-            assert l == ['call', 'return', 'call', 'c_call', 'c_return', 'return']
-            if isinstance(seen[0], Method):
-                w_class = space.type(seen[0].w_instance)
+            assert l[-6:] == ['call', 'return', 'call', 'c_call', 'c_return', 'return']
+            if isinstance(seen[-1], Method):
+                w_class = space.type(seen[-1].w_instance)
                 found = 'method %s of %s' % (
-                    seen[0].w_function.name,
+                    seen[-1].w_function.name,
                     w_class.getname(space))
             else:
-                assert isinstance(seen[0], Function)
-                found = 'builtin %s' % seen[0].name
+                assert isinstance(seen[-1], Function)
+                found = 'builtin %s' % seen[-1].name
             assert found == expected_c_call
 
         check_snippet('l = []; l.append(42)', 'method append of list')
@@ -210,7 +210,7 @@
             return
             """ % snippet)
             space.getexecutioncontext().setllprofile(None, None)
-            assert l == ['call', 'return', 'call', 'c_call', 'c_exception', 'return']
+            assert l[-6:] == ['call', 'return', 'call', 'c_call', 'c_exception', 'return']
 
         check_snippet('d = {}; d.__getitem__(42)')
 
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -153,6 +153,8 @@
         r"""
         seen = []
         def tracer(f, event, *args):
+            if f.f_code.co_name == "decode":
+                return tracer
             seen.append((event, f.f_lineno))
             if len(seen) == 5:
                 f.f_lineno = 1       # bug shown only when setting lineno to 1
@@ -297,7 +299,8 @@
 
         l = []
         def trace(a,b,c):
-            l.append((a,b,c))
+            if a.f_code.co_name != "decode":
+                l.append((a,b,c))
 
         def f():
             h = _testing.Hidden()
diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py
--- a/pypy/interpreter/test/test_syntax.py
+++ b/pypy/interpreter/test/test_syntax.py
@@ -691,6 +691,25 @@
                                     "bye" : 5, "kw" : 6, "return" : 42}
         """
 
+    def test_bug_annotations_lambda(self):
+        """
+        # those used to crash
+        def broken(*a: lambda x: None):
+            pass
+
+        def broken(**a: lambda x: None):
+            pass
+        """
+
+    def test_bug_annotation_inside_nested_function(self):
+        """
+        # this used to crash
+        def f1():
+            def f2(*args: int):
+                pass
+        f1()
+        """
+
 class AppTestSyntaxError:
 
     def test_tokenizer_error_location(self):
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -193,8 +193,12 @@
 
 @specialize.arg(2)
 def min_max(space, args, implementation_of):
-    if not jit.we_are_jitted() or len(args.arguments_w) != 1 and \
-            jit.loop_unrolling_heuristic(args.arguments_w, len(args.arguments_w)):
+    # the 'normal' version includes a JIT merge point, which will make a
+    # new loop (from the interpreter or from another JIT loop).  If we
+    # give exactly two arguments to the call to max(), or a JIT virtual
+    # list of arguments, then we pick the 'unroll' version with no JIT
+    # merge point.
+    if jit.isvirtual(args.arguments_w) or len(args.arguments_w) == 2:
         return min_max_unroll(space, args, implementation_of)
     else:
         return min_max_normal(space, args, implementation_of)
diff --git a/pypy/module/_cffi_backend/errorbox.py b/pypy/module/_cffi_backend/errorbox.py
--- a/pypy/module/_cffi_backend/errorbox.py
+++ b/pypy/module/_cffi_backend/errorbox.py
@@ -69,7 +69,10 @@
                 import sys
                 class FileLike:
                     def write(self, x):
-                        of.write(x)
+                        try:
+                            of.write(x)
+                        except:
+                            pass
                         self.buf += x
                 fl = FileLike()
                 fl.buf = ''
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3935,8 +3935,8 @@
 
 def test_char_pointer_conversion():
     import warnings
-    assert __version__.startswith(("1.8", "1.9", "1.10", "1.11", "1.12")), (
-        "consider turning the warning into an error")
+    assert __version__.startswith("1."), (
+        "the warning will be an error if we ever release cffi 2.x")
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))
     BVoidP = new_pointer_type(new_void_type())
diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py
--- a/pypy/module/_cppyy/capi/loadable_capi.py
+++ b/pypy/module/_cppyy/capi/loadable_capi.py
@@ -1,13 +1,18 @@
 import os
+
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib import jit, jit_libffi, libffi, rdynload, objectmodel
 from rpython.rlib.rarithmetic import r_singlefloat
 from rpython.tool import leakfinder
 
-from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.error import oefmt
+from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.argument import Arguments
+from pypy.interpreter.gateway import interp2app, interpindirect2app
+from pypy.interpreter.typedef import TypeDef
+from pypy.objspace.std.iterobject import W_AbstractSeqIterObject
 
+from pypy.module._rawffi.array import W_ArrayInstance
 from pypy.module._cffi_backend import ctypefunc, ctypeprim, cdataobj, misc
 from pypy.module._cffi_backend import newtype
 from pypy.module._cppyy import ffitypes
@@ -23,10 +28,11 @@
 
 class _Arg:         # poor man's union
     _immutable_ = True
-    def __init__(self, tc, h = 0, l = -1, s = '', p = rffi.cast(rffi.VOIDP, 0)):
+    def __init__(self, tc, h = 0, l = -1, d = -1., s = '', p = rffi.cast(rffi.VOIDP, 0)):
         self.tc      = tc
         self._handle = h
         self._long   = l
+        self._double = d
         self._string = s
         self._voidp  = p
 
@@ -40,6 +46,11 @@
     def __init__(self, val):
         _Arg.__init__(self, 'l', l = val)
 
+class _ArgD(_Arg):
+    _immutable_ = True
+    def __init__(self, val):
+        _Arg.__init__(self, 'd', d = val)
+
 class _ArgS(_Arg):
     _immutable_ = True
     def __init__(self, val):
@@ -89,6 +100,9 @@
                     assert obj._voidp != rffi.cast(rffi.VOIDP, 0)
                     data = rffi.cast(rffi.VOIDPP, data)
                     data[0] = obj._voidp
+                elif obj.tc == 'd':
+                    assert isinstance(argtype, ctypeprim.W_CTypePrimitiveFloat)
+                    misc.write_raw_float_data(data, rffi.cast(rffi.DOUBLE, obj._double), argtype.size)
                 else:    # only other use is string
                     assert obj.tc == 's'
                     n = len(obj._string)
@@ -182,6 +196,7 @@
             'call_f'       : ([c_method, c_object, c_int, c_voidp],   c_float),
             'call_d'       : ([c_method, c_object, c_int, c_voidp],   c_double),
             'call_ld'      : ([c_method, c_object, c_int, c_voidp],   c_ldouble),
+            'call_nld'     : ([c_method, c_object, c_int, c_voidp],   c_double),
 
             'call_r'       : ([c_method, c_object, c_int, c_voidp],   c_voidp),
             # call_s actually takes an size_t* as last parameter, but this will do
@@ -236,6 +251,8 @@
             'method_prototype'         : ([c_scope, c_method, c_int], c_ccharp),
             'is_const_method'          : ([c_method],                 c_int),
 
+            'get_num_templated_methods': ([c_scope],                  c_int),
+            'get_templated_method_name': ([c_scope, c_index],         c_ccharp),
             'exists_method_template'   : ([c_scope, c_ccharp],        c_int),
             'method_is_template'       : ([c_scope, c_index],         c_int),
             'get_method_template'      : ([c_scope, c_ccharp, c_ccharp],        c_method),
@@ -272,9 +289,11 @@
             'stdstring2charp'          : ([c_object, c_voidp],        c_ccharp),
             'stdstring2stdstring'      : ([c_object],                 c_object),
 
-            'stdvector_valuetype'      : ([c_ccharp],                 c_ccharp),
-            'stdvector_valuesize'      : ([c_ccharp],                 c_size_t),
+            'longdouble2double'        : ([c_voidp],                  c_double),
+            'double2longdouble'        : ([c_double, c_voidp],        c_void),
 
+            'vectorbool_getitem'       : ([c_object, c_int],          c_int),
+            'vectorbool_setitem'       : ([c_object, c_int, c_int],   c_void),
         }
 
         # size/offset are backend-specific but fixed after load
@@ -401,7 +420,9 @@
     return rffi.cast(rffi.DOUBLE, space.float_w(call_capi(space, 'call_d', args)))
 def c_call_ld(space, cppmethod, cppobject, nargs, cargs):
     args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs)]
-    return rffi.cast(rffi.LONGDOUBLE, space.float_w(call_capi(space, 'call_ld', args)))
+    #return rffi.cast(rffi.LONGDOUBLE, space.float_w(call_capi(space, 'call_ld', args)))
+    # call_nld narrows long double to double
+    return rffi.cast(rffi.DOUBLE, space.float_w(call_capi(space, 'call_nld', args)))
 
 def c_call_r(space, cppmethod, cppobject, nargs, cargs):
     args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs)]
@@ -561,16 +582,21 @@
 def c_is_const_method(space, cppmeth):
     return space.bool_w(call_capi(space, 'is_const_method', [_ArgH(cppmeth)]))
 
+def c_get_num_templated_methods(space, cppscope):
+    return space.int_w(call_capi(space, 'method_is_template', [_ArgH(cppscope.handle)]))
+def c_get_templated_method_name(space, cppscope, index):
+    args = [_ArgH(cppscope.handle), _ArgL(index)]
+    return charp2str_free(space, call_capi(space, 'method_is_template', args))
 def c_exists_method_template(space, cppscope, name):
     args = [_ArgH(cppscope.handle), _ArgS(name)]
     return space.bool_w(call_capi(space, 'exists_method_template', args))
 def c_method_is_template(space, cppscope, index):
     args = [_ArgH(cppscope.handle), _ArgL(index)]
     return space.bool_w(call_capi(space, 'method_is_template', args))
-
 def c_get_method_template(space, cppscope, name, proto):
     args = [_ArgH(cppscope.handle), _ArgS(name), _ArgS(proto)]
     return rffi.cast(C_METHOD, space.uint_w(call_capi(space, 'get_method_template', args)))
+
 def c_get_global_operator(space, nss, lc, rc, op):
     if nss is not None:
         args = [_ArgH(nss.handle), _ArgH(lc.handle), _ArgH(rc.handle), _ArgS(op)]
@@ -619,7 +645,7 @@
     return space.bool_w(call_capi(space, 'is_enum_data', args))
 def c_get_dimension_size(space, cppscope, datamember_index, dim_idx):
     args = [_ArgH(cppscope.handle), _ArgL(datamember_index), _ArgL(dim_idx)]
-    return space.bool_w(call_capi(space, 'get_dimension_size', args))
+    return space.int_w(call_capi(space, 'get_dimension_size', args))
 
 # misc helpers ---------------------------------------------------------------
 def c_strtoll(space, svalue):
@@ -650,24 +676,94 @@
 def c_stdstring2stdstring(space, cppobject):
     return _cdata_to_cobject(space, call_capi(space, 'stdstring2stdstring', [_ArgH(cppobject)]))
 
-def c_stdvector_valuetype(space, pystr):
-    return charp2str_free(space, call_capi(space, 'stdvector_valuetype', [_ArgS(pystr)]))
+def c_longdouble2double(space, addr):
+    return space.float_w(call_capi(space, 'longdouble2double', [_ArgP(addr)]))
+def c_double2longdouble(space, dval, addr):
+    call_capi(space, 'double2longdouble', [_ArgD(dval), _ArgP(addr)])
 
-def c_stdvector_valuetype(space, pystr):
-    return charp2str_free(space, call_capi(space, 'stdvector_valuetype', [_ArgS(pystr)]))
-def c_stdvector_valuesize(space, pystr):
-    return _cdata_to_size_t(space, call_capi(space, 'stdvector_valuesize', [_ArgS(pystr)]))
+def c_vectorbool_getitem(space, vbool, idx):
+    return call_capi(space, 'vectorbool_getitem', [_ArgH(vbool), _ArgL(idx)])
+def c_vectorbool_setitem(space, vbool, idx, value):
+    call_capi(space, 'vectorbool_setitem', [_ArgH(vbool), _ArgL(idx), _ArgL(value)])
 
 
 # TODO: factor these out ...
 # pythonizations
 def stdstring_c_str(space, w_self):
     """Return a python string taking into account \0"""
-
     from pypy.module._cppyy import interp_cppyy
     cppstr = space.interp_w(interp_cppyy.W_CPPInstance, w_self, can_be_None=False)
     return space.newtext(c_stdstring2charp(space, cppstr._rawobject))
 
+def vbool_getindex(space, w_vbool, w_idx):
+    idx = space.getindex_w(w_idx, space.w_IndexError, "std::vector<bool> index")
+    sz = space.len_w(w_vbool)
+    if idx < 0: idx += sz
+    if idx < 0 or idx >= sz:
+        raise IndexError
+    return idx
+
+def vectorbool_getitem(space, w_self, w_idx):
+    """Index a std::vector<bool>, return the value"""
+    from pypy.module._cppyy import interp_cppyy
+    vbool = space.interp_w(interp_cppyy.W_CPPInstance, w_self, can_be_None=False)
+    idx = vbool_getindex(space, w_self, w_idx)
+    item = c_vectorbool_getitem(space, vbool._rawobject, idx)
+    return space.newbool(space.is_true(item))
+
+def vectorbool_setitem(space, w_self, w_idx, w_value):
+    """Index a std::vector<bool>, set the value"""
+    from pypy.module._cppyy import interp_cppyy
+    vbool = space.interp_w(interp_cppyy.W_CPPInstance, w_self, can_be_None=False)
+    idx = vbool_getindex(space, w_self, w_idx)
+    c_vectorbool_setitem(space, vbool._rawobject, idx, int(space.is_true(w_value)))
+
+class W_STLVectorIter(W_AbstractSeqIterObject):
+    # w_seq and index are in base class
+    _immutable_fields_ = ['converter', 'data', 'len', 'stride']
+
+    def __init__(self, space, w_vector):
+        W_AbstractSeqIterObject.__init__(self, w_vector)
+        # TODO: this should live in rpythonize.py or something so that the
+        # imports can move to the top w/o getting circles
+        from pypy.module._cppyy import interp_cppyy
+        assert isinstance(w_vector, interp_cppyy.W_CPPInstance)
+        vector = space.interp_w(interp_cppyy.W_CPPInstance, w_vector)
+
+        v_type = c_resolve_name(space, vector.clsdecl.name+'::value_type')
+        v_size = c_size_of_type(space, v_type)
+
+        if not v_type or not v_size:
+            raise NotImplementedError   # fallback on getitem
+
+        from pypy.module._cppyy import converter
+        self.converter = converter.get_converter(space, v_type, '')
+
+        # this 'data' is from the decl, so not the pythonized data from pythonify.py
+        w_arr = space.call_obj_args(vector.clsdecl.get_overload('data'), w_vector, Arguments(space, []))
+        arr = space.interp_w(W_ArrayInstance, w_arr, can_be_None=True)
+        if not arr:
+            raise OperationError(space.w_StopIteration, space.w_None)
+
+        self.data    = rffi.cast(rffi.CCHARP, space.uint_w(arr.getbuffer(space)))
+        self.len     = space.uint_w(space.call_obj_args(vector.clsdecl.get_overload('size'), w_vector, Arguments(space, [])))
+        self.stride  = v_size
+
+    def descr_next(self, space):
+        if self.w_seq is None:
+            raise OperationError(space.w_StopIteration, space.w_None)
+        if self.len <= self.index:
+            self.w_seq = None
+            raise OperationError(space.w_StopIteration, space.w_None)
+        offset = lltype.direct_ptradd(self.data, rffi.cast(rffi.SIZE_T, self.index*self.stride))
+        w_item = self.converter.from_memory(space, space.w_None, rffi.cast(rffi.LONG, offset))
+        self.index += 1
+        return w_item
+
+def stdvector_iter(space, w_self):
+    return W_STLVectorIter(space, w_self)
+
+
 # setup pythonizations for later use at run-time
 _pythonizations = {}
 def register_pythonizations(space):
@@ -678,6 +774,12 @@
         ### std::string
         stdstring_c_str,
 
+        ### std::vector
+        stdvector_iter,
+
+        ### std::vector<bool>
+        vectorbool_getitem,
+        vectorbool_setitem,
     ]
 
     for f in allfuncs:
@@ -692,3 +794,10 @@
         space.setattr(w_pycppclass, space.newtext("c_str"), _pythonizations["stdstring_c_str"])
         _method_alias(space, w_pycppclass, "_cppyy_as_builtin", "c_str")
         _method_alias(space, w_pycppclass, "__str__",           "c_str")
+
+    if  name.find("std::vector<bool", 0, 16) == 0:
+        space.setattr(w_pycppclass, space.newtext("__getitem__"), _pythonizations["vectorbool_getitem"])
+        space.setattr(w_pycppclass, space.newtext("__setitem__"), _pythonizations["vectorbool_setitem"])
+
+    elif name.find("std::vector", 0, 11) == 0:
+        space.setattr(w_pycppclass, space.newtext("__iter__"), _pythonizations["stdvector_iter"])
diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -1,15 +1,12 @@
 import sys
 
 from pypy.interpreter.error import OperationError, oefmt
-
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.rarithmetic import r_singlefloat, r_longfloat
 from rpython.rlib import rfloat, rawrefcount
-
 from pypy.module._rawffi.interp_rawffi import letter2tp
 from pypy.module._rawffi.array import W_ArrayInstance
-
-from pypy.module._cppyy import helper, capi, ffitypes
+from pypy.module._cppyy import helper, capi, ffitypes, lowlevelviews
 
 # Converter objects are used to translate between RPython and C++. They are
 # defined by the type name for which they provide conversion. Uses are for
@@ -118,7 +115,7 @@
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         self._is_abstract(space)
 
     def to_memory(self, space, w_obj, w_value, offset):
@@ -145,11 +142,12 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         # read access, so no copy needed
         address_value = self._get_raw_address(space, w_obj, offset)
         address = rffi.cast(rffi.ULONG, address_value)
-        return W_ArrayInstance(space, letter2tp(space, self.typecode), self.size, address)
+        return lowlevelviews.W_LowLevelView(
+            space, letter2tp(space, self.typecode), self.size, address)
 
     def to_memory(self, space, w_obj, w_value, offset):
         # copy the full array (uses byte copy for now)
@@ -186,11 +184,12 @@
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'o'
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         # read access, so no copy needed
         address_value = self._get_raw_address(space, w_obj, offset)
         address = rffi.cast(rffi.ULONGP, address_value)
-        return W_ArrayInstance(space, letter2tp(space, self.typecode), self.size, address[0])
+        return lowlevelviews.W_LowLevelView(
+            space, letter2tp(space, self.typecode), self.size, address[0])
 
     def to_memory(self, space, w_obj, w_value, offset):
         # copy only the pointer value
@@ -212,10 +211,13 @@
         x[0] = self._unwrap_object(space, w_obj)
 
     def default_argument_libffi(self, space, address):
+        if not self.valid_default:
+            from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
+            raise FastCallNotPossible
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self.default
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = self._get_raw_address(space, w_obj, offset)
         rffiptr = rffi.cast(self.c_ptrtype, address)
         return self._wrap_object(space, rffiptr[0])
@@ -225,7 +227,7 @@
         rffiptr = rffi.cast(self.c_ptrtype, address)
         rffiptr[0] = self._unwrap_object(space, w_value)
 
-class ConstRefNumericTypeConverterMixin(NumericTypeConverterMixin):
+class ConstRefNumericTypeConverterMixin(object):
     _mixin_ = True
 
     def cffi_type(self, space):
@@ -284,7 +286,7 @@
         x = rffi.cast(rffi.LONGP, address)
         x[0] = self._unwrap_object(space, w_obj)
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = rffi.cast(rffi.CCHARP, self._get_raw_address(space, w_obj, offset))
         if address[0] == '\x01':
             return space.w_True
@@ -309,7 +311,7 @@
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = rffi.cast(rffi.CCHARP, self._get_raw_address(space, w_obj, offset))
         return space.newbytes(address[0])
 
@@ -322,59 +324,92 @@
     pass
 
 class FloatConverter(ffitypes.typeid(rffi.FLOAT), FloatTypeConverterMixin, TypeConverter):
-    _immutable_fields_ = ['default']
+    _immutable_fields_ = ['default', 'valid_default']
 
     def __init__(self, space, default):
-        if default:
+        self.valid_default = False
+        try:
             fval = float(rfloat.rstring_to_float(default))
-        else:
+            self.valid_default = True
+        except Exception:
             fval = float(0.)
-        self.default = r_singlefloat(fval)
+        self.default = rffi.cast(rffi.FLOAT, r_singlefloat(fval))
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = self._get_raw_address(space, w_obj, offset)
         rffiptr = rffi.cast(self.c_ptrtype, address)
         return self._wrap_object(space, rffiptr[0])
 
-class ConstFloatRefConverter(FloatConverter):
+class ConstFloatRefConverter(ConstRefNumericTypeConverterMixin, FloatConverter):
     _immutable_fields_ = ['typecode']
     typecode = 'f'
 
-    def cffi_type(self, space):
-        state = space.fromcache(ffitypes.State)
-        return state.c_voidp
-
-    def convert_argument_libffi(self, space, w_obj, address, scratch):
-        from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
-        raise FastCallNotPossible
-
 class DoubleConverter(ffitypes.typeid(rffi.DOUBLE), FloatTypeConverterMixin, TypeConverter):
-    _immutable_fields_ = ['default']
+    _immutable_fields_ = ['default', 'valid_default']
 
     def __init__(self, space, default):
-        if default:
+        self.valid_default = False
+        try:
             self.default = rffi.cast(self.c_type, rfloat.rstring_to_float(default))
-        else:
+            self.valid_default = True
+        except Exception:
             self.default = rffi.cast(self.c_type, 0.)
 
 class ConstDoubleRefConverter(ConstRefNumericTypeConverterMixin, DoubleConverter):
     _immutable_fields_ = ['typecode']
     typecode = 'd'
 
-class LongDoubleConverter(ffitypes.typeid(rffi.LONGDOUBLE), FloatTypeConverterMixin, TypeConverter):
-    _immutable_fields_ = ['default']
+class LongDoubleConverter(TypeConverter):
+    _immutable_fields_ = ['default', 'valid_default']
+    typecode = 'g'
 
     def __init__(self, space, default):
-        if default:
-            fval = float(rfloat.rstring_to_float(default))
-        else:
-            fval = float(0.)
-        self.default = r_longfloat(fval)
+        self.valid_default = False
+        try:
+            # use float() instead of cast with r_longfloat
+            fval = rffi.cast(rffi.DOUBLE, rfloat.rstring_to_float(default))
+            self.valid_default = True
+        except Exception:
+            fval = rffi.cast(rffi.DOUBLE, 0.)
+        #self.default = r_longfloat(fval)
+        self.default = fval
+
+    def convert_argument(self, space, w_obj, address):
+        x = rffi.cast(rffi.VOIDP, address)
+        capi.c_double2longdouble(space, space.float_w(w_obj), x)
+        ba = rffi.cast(rffi.CCHARP, address)
+        ba[capi.c_function_arg_typeoffset(space)] = self.typecode
+
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
+        x = rffi.cast(rffi.VOIDP, address)
+        capi.c_double2longdouble(space, space.float_w(w_obj), x)
+
+    def default_argument_libffi(self, space, address):
+        if not self.valid_default:
+            from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
+            raise FastCallNotPossible
+        x = rffi.cast(rffi.VOIDP, address)
+        capi.c_double2longdouble(space, self.default, x)
+
+    def from_memory(self, space, w_obj, offset):
+        address = self._get_raw_address(space, w_obj, offset)
+        rffiptr = rffi.cast(rffi.VOIDP, address)
+        return space.newfloat(capi.c_longdouble2double(space, rffiptr))
+
+    def to_memory(self, space, w_obj, w_value, offset):
+        address = self._get_raw_address(space, w_obj, offset)
+        rffiptr = rffi.cast(rffi.VOIDP, address)
+        capi.c_double2longdouble(space, space.float_w(w_value), rffiptr)
 
 class ConstLongDoubleRefConverter(ConstRefNumericTypeConverterMixin, LongDoubleConverter):
     _immutable_fields_ = ['typecode']
     typecode = 'g'
 
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
+        capi.c_double2longdouble(space, space.float_w(w_obj), rffi.cast(rffi.VOIDP, scratch))
+        x = rffi.cast(rffi.VOIDPP, address)
+        x[0] = scratch
+
 
 class CStringConverter(TypeConverter):
     def convert_argument(self, space, w_obj, address):
@@ -384,7 +419,7 @@
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'p'
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = self._get_raw_address(space, w_obj, offset)
         charpptr = rffi.cast(rffi.CCHARPP, address)
         return space.newtext(rffi.charp2str(charpptr[0]))
@@ -398,13 +433,15 @@
     def __init__(self, space, extra):
         self.size = extra
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = self._get_raw_address(space, w_obj, offset)
         charpptr = rffi.cast(rffi.CCHARP, address)
-        strsize = self.size
-        if charpptr[self.size-1] == '\0':
-            strsize = self.size-1  # rffi will add \0 back
-        return space.newtext(rffi.charpsize2str(charpptr, strsize))
+        if 0 <= self.size and self.size != 2**31-1:   # cling's code for "unknown" (?)
+            strsize = self.size
+            if charpptr[self.size-1] == '\0':
+                strsize = self.size-1  # rffi will add \0 back
+            return space.newtext(rffi.charpsize2str(charpptr, strsize))
+        return space.newtext(rffi.charp2str(charpptr))
 
 
 class VoidPtrConverter(TypeConverter):
@@ -429,7 +466,7 @@
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = self._unwrap_object(space, w_obj)
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         # returned as a long value for the address (INTPTR_T is not proper
         # per se, but rffi does not come with a PTRDIFF_T)
         address = self._get_raw_address(space, w_obj, offset)
@@ -438,7 +475,7 @@
             from pypy.module._cppyy import interp_cppyy
             return interp_cppyy.get_nullptr(space)
         shape = letter2tp(space, 'P')
-        return W_ArrayInstance(space, shape, sys.maxint/shape.size, ptrval)
+        return lowlevelviews.W_LowLevelView(space, shape, sys.maxint/shape.size, ptrval)
 
     def to_memory(self, space, w_obj, w_value, offset):
         address = rffi.cast(rffi.VOIDPP, self._get_raw_address(space, w_obj, offset))
@@ -491,7 +528,7 @@
         from pypy.module._cppyy.interp_cppyy import W_CPPInstance
         if isinstance(w_obj, W_CPPInstance):
             from pypy.module._cppyy.interp_cppyy import INSTANCE_FLAGS_IS_RVALUE
-            if w_obj.flags & INSTANCE_FLAGS_IS_RVALUE:
+            if w_obj.rt_flags & INSTANCE_FLAGS_IS_RVALUE:
                 # reject moves as all are explicit
                 raise ValueError("lvalue expected")
             if capi.c_is_subtype(space, w_obj.clsdecl, self.clsdecl):
@@ -523,14 +560,14 @@
         from pypy.module._cppyy.interp_cppyy import W_CPPInstance, INSTANCE_FLAGS_IS_RVALUE
         obj = space.interp_w(W_CPPInstance, w_obj)
         if obj:
-            if obj.flags & INSTANCE_FLAGS_IS_RVALUE:
-                obj.flags &= ~INSTANCE_FLAGS_IS_RVALUE
+            if obj.rt_flags & INSTANCE_FLAGS_IS_RVALUE:
+                obj.rt_flags &= ~INSTANCE_FLAGS_IS_RVALUE
                 try:
                     return InstanceRefConverter._unwrap_object(self, space, w_obj)
                 except Exception:
                     # TODO: if the method fails on some other converter, then the next
                     # overload can not be an rvalue anymore
-                    obj.flags |= INSTANCE_FLAGS_IS_RVALUE
+                    obj.rt_flags |= INSTANCE_FLAGS_IS_RVALUE
                     raise
         raise oefmt(space.w_ValueError, "object is not an rvalue")
 
@@ -541,7 +578,7 @@
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible       # TODO: by-value is a jit_libffi special case
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
         from pypy.module._cppyy import interp_cppyy
         return interp_cppyy.wrap_cppinstance(space, address, self.clsdecl, do_cast=False)
@@ -561,15 +598,11 @@
                 return capi.C_NULL_OBJECT
             raise e
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
         from pypy.module._cppyy import interp_cppyy
         return interp_cppyy.wrap_cppinstance(space, address, self.clsdecl, do_cast=False)
 
-    def to_memory(self, space, w_obj, w_value, offset):
-        address = rffi.cast(rffi.VOIDPP, self._get_raw_address(space, w_obj, offset))
-        address[0] = rffi.cast(rffi.VOIDP, self._unwrap_object(space, w_value))
-
 class InstancePtrPtrConverter(InstancePtrConverter):
     typecode = 'o'
 
@@ -591,12 +624,31 @@
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
         from pypy.module._cppyy import interp_cppyy
         return interp_cppyy.wrap_cppinstance(
             space, address, self.clsdecl, do_cast=False, is_ref=True)
 
+    def to_memory(self, space, w_obj, w_value, offset):
+        # the actual data member is of object* type, but we receive a pointer to that
+        # data member in order to modify its value, so by convention, the internal type
+        # used is object**
+        address = rffi.cast(rffi.VOIDPP, self._get_raw_address(space, w_obj, offset))
+        from pypy.module._cppyy.interp_cppyy import W_CPPInstance
+        cppinstance = space.interp_w(W_CPPInstance, w_value, can_be_None=True)
+        if cppinstance:
+            rawobject = cppinstance.get_rawobject()
+            offset = capi.c_base_offset(space, cppinstance.clsdecl, self.clsdecl, rawobject, 1)
+            obj_address = capi.direct_ptradd(rawobject, offset)
+            address[0] = rffi.cast(rffi.VOIDP, obj_address);
+            # register the value for potential recycling
+            from pypy.module._cppyy.interp_cppyy import memory_regulator
+            memory_regulator.register(cppinstance)
+        else:
+            raise oefmt(space.w_TypeError,
+                "cannot pass %T instance as %s", w_value, self.clsdecl.name)
+
     def finalize_call(self, space, w_obj):
         if self.ref_buffer:
             set_rawobject(space, w_obj, self.ref_buffer[0])
@@ -606,8 +658,27 @@
             lltype.free(self.ref_buffer, flavor='raw')
             self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
 
+class InstanceArrayConverter(InstancePtrConverter):
+    _immutable_fields_ = ['size']
+
+    def __init__(self, space, clsdecl, array_size, dimensions):
+        InstancePtrConverter.__init__(self, space, clsdecl)
+        if array_size <= 0 or array_size == 2**31-1:   # cling's code for "unknown" (?)
+            self.size = sys.maxint
+        else:
+            self.size = array_size
+        # peel one off as that should be the same as the array size
+        self.dimensions = dimensions[1:]
+
+    def from_memory(self, space, w_obj, offset):
+        address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
+        return lowlevelviews.W_ArrayOfInstances(space, self.clsdecl, address, self.size, self.dimensions)
+
+    def to_memory(self, space, w_obj, w_value, offset):
+        self._is_abstract(space)
+
+
 class StdStringConverter(InstanceConverter):
-
     def __init__(self, space, extra):
         from pypy.module._cppyy import interp_cppyy
         cppclass = interp_cppyy.scope_byname(space, capi.std_string_name)
@@ -633,6 +704,34 @@
     def free_argument(self, space, arg):
         capi.c_destruct(space, self.clsdecl, rffi.cast(capi.C_OBJECT, rffi.cast(rffi.VOIDPP, arg)[0]))
 
+class StdStringMoveConverter(StdStringConverter):
+    def _unwrap_object(self, space, w_obj):
+        # moving is same as by-ref, but have to check that move is allowed
+        moveit_reason = 3
+        from pypy.module._cppyy.interp_cppyy import W_CPPInstance, INSTANCE_FLAGS_IS_RVALUE
+        try:
+            obj = space.interp_w(W_CPPInstance, w_obj)
+            if obj and obj.rt_flags & INSTANCE_FLAGS_IS_RVALUE:
+                obj.rt_flags &= ~INSTANCE_FLAGS_IS_RVALUE
+                moveit_reason = 1
+            else:
+                moveit_reason = 0
+        except:
+            pass
+
+        if moveit_reason:
+            try:
+                return StdStringConverter._unwrap_object(self, space, w_obj)
+            except Exception:
+                 if moveit_reason == 1:
+                    # TODO: if the method fails on some other converter, then the next
+                    # overload can not be an rvalue anymore
+                    obj = space.interp_w(W_CPPInstance, w_obj)
+                    obj.rt_flags |= INSTANCE_FLAGS_IS_RVALUE
+                    raise
+
+        raise oefmt(space.w_ValueError, "object is not an rvalue")
+
 class StdStringRefConverter(InstancePtrConverter):
     _immutable_fields_ = ['cppclass', 'typecode']
     typecode    = 'V'
@@ -749,7 +848,7 @@
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = self.typecode
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
         from pypy.module._cppyy import interp_cppyy
         return interp_cppyy.wrap_cppinstance(space, address,
@@ -758,7 +857,7 @@
 class SmartPtrPtrConverter(SmartPtrConverter):
     typecode    = 'o'
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         self._is_abstract(space)
 
     def to_memory(self, space, w_obj, w_value, offset):
@@ -770,7 +869,7 @@
 
 
 class MacroConverter(TypeConverter):
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         # TODO: get the actual type info from somewhere ...
         address = self._get_raw_address(space, w_obj, offset)
         longptr = rffi.cast(rffi.LONGP, address)
@@ -803,20 +902,28 @@
         pass
 
     # match of decorated, unqualified type
-    compound = helper.compound(name)
+    cpd = helper.compound(name)
     clean_name = capi.c_resolve_name(space, helper.clean_type(name))
     try:
-        return _converters[clean_name+compound](space, default)
+        return _converters[clean_name+cpd](space, default)
     except KeyError:
         pass
 
-    # arrays
+    # arrays (array_size may be negative, meaning: no size or no size found)
+    array_size = -1
+    if cpd == "[]":
+        array_size = helper.array_size(_name)    # uses original arg
+    elif cpd == '*' and ':' in default:
+        # this happens for multi-dimensional arrays: those are described as pointers
+        cpd = "[]"
+        splitpos = default.find(':')
+        if 0 < splitpos:     # always true, but needed for annotator
+            array_size = int(default[:splitpos])
+
     try:
-        # array_index may be negative to indicate no size or no size found
-        array_size = helper.array_size(_name)     # uses original arg
         # TODO: using clean_name here drops const (e.g. const char[] will
         # never be seen this way)
-        return _a_converters[clean_name+compound](space, array_size)
+        return _a_converters[clean_name+cpd](space, array_size)
     except KeyError:
         pass
 
@@ -830,24 +937,28 @@
         # check smart pointer type
         check_smart = capi.c_smartptr_info(space, clean_name)
         if check_smart[0]:
-            if compound == '':
+            if cpd == '':
                 return SmartPtrConverter(space, clsdecl, check_smart[1], check_smart[2])
-            elif compound == '*':
+            elif cpd == '*':
                 return SmartPtrPtrConverter(space, clsdecl, check_smart[1], check_smart[2])
-            elif compound == '&':
+            elif cpd == '&':
                 return SmartPtrRefConverter(space, clsdecl, check_smart[1], check_smart[2])
             # fall through: can still return smart pointer in non-smart way
 
         # type check for the benefit of the annotator
-        if compound == "*":
+        if cpd == "*":
             return InstancePtrConverter(space, clsdecl)
-        elif compound == "&":
+        elif cpd == "&":
             return InstanceRefConverter(space, clsdecl)
-        elif compound == "&&":
+        elif cpd == "&&":
             return InstanceMoveConverter(space, clsdecl)
-        elif compound == "**":
+        elif cpd in ["**", "*[]", "&*"]:
             return InstancePtrPtrConverter(space, clsdecl)
-        elif compound == "":
+        elif cpd == "[]" and array_size > 0:
+            # default encodes the dimensions
+            dims = default.split(':')
+            return InstanceArrayConverter(space, clsdecl, array_size, dims)
+        elif cpd == "":
             return InstanceConverter(space, clsdecl)
     elif "(anonymous)" in name:
         # special case: enum w/o a type name
@@ -859,7 +970,7 @@
             return FunctionPointerConverter(space, name[pos+2:])
 
     # void* or void converter (which fails on use)
-    if 0 <= compound.find('*'):
+    if 0 <= cpd.find('*'):
         return VoidPtrConverter(space, default)  # "user knows best"
 
     # return a void converter here, so that the class can be build even
@@ -874,8 +985,8 @@
 _converters["const float&"]             = ConstFloatRefConverter
 _converters["double"]                   = DoubleConverter
 _converters["const double&"]            = ConstDoubleRefConverter
-#_converters["long double"]              = LongDoubleConverter
-#_converters["const long double&"]       = ConstLongDoubleRefConverter
+_converters["long double"]              = LongDoubleConverter
+_converters["const long double&"]       = ConstLongDoubleRefConverter
 _converters["const char*"]              = CStringConverter
 _converters["void*"]                    = VoidPtrConverter
 _converters["void**"]                   = VoidPtrPtrConverter
@@ -885,6 +996,7 @@
 _converters["std::basic_string<char>"]           = StdStringConverter
 _converters["const std::basic_string<char>&"]    = StdStringConverter     # TODO: shouldn't copy
 _converters["std::basic_string<char>&"]          = StdStringRefConverter
+_converters["std::basic_string<char>&&"]         = StdStringMoveConverter
 
 _converters["PyObject*"]                         = PyObjectConverter
 
@@ -908,7 +1020,12 @@
             _immutable_ = True
             typecode = c_tc
             def __init__(self, space, default):
-                self.default = rffi.cast(self.c_type, capi.c_strtoll(space, default))
+                self.valid_default = False
+                try:
+                    self.default = rffi.cast(self.c_type, capi.c_strtoll(space, default))
+                    self.valid_default = True
+                except Exception:
+                    self.default = rffi.cast(self.c_type, 0)
         class ConstRefConverter(ConstRefNumericTypeConverterMixin, BasicConverter):
             _immutable_ = True
         for name in names:
@@ -925,7 +1042,12 @@
             _immutable_ = True
             typecode = c_tc
             def __init__(self, space, default):
-                self.default = rffi.cast(self.c_type, capi.c_strtoll(space, default))
+                self.valid_default = False
+                try:
+                    self.default = rffi.cast(self.c_type, capi.c_strtoll(space, default))
+                    self.valid_default = True
+                except Exception:
+                    self.default = rffi.cast(self.c_type, 0)
         class ConstRefConverter(ConstRefNumericTypeConverterMixin, BasicConverter):
             _immutable_ = True
         for name in names:
@@ -945,7 +1067,12 @@
             _immutable_ = True
             typecode = c_tc
             def __init__(self, space, default):
-                self.default = rffi.cast(self.c_type, capi.c_strtoull(space, default))
+                self.valid_default = False
+                try:
+                    self.default = rffi.cast(self.c_type, capi.c_strtoull(space, default))
+                    self.valid_default = True
+                except Exception:
+                    self.default = rffi.cast(self.c_type, 0)
         class ConstRefConverter(ConstRefNumericTypeConverterMixin, BasicConverter):
             _immutable_ = True
         for name in names:
@@ -1002,6 +1129,7 @@
         ("std::basic_string<char>",         "string"),
         ("const std::basic_string<char>&",  "const string&"),
         ("std::basic_string<char>&",        "string&"),
+        ("std::basic_string<char>&&",       "string&&"),
 
         ("PyObject*",                       "_object*"),
     )
diff --git a/pypy/module/_cppyy/executor.py b/pypy/module/_cppyy/executor.py
--- a/pypy/module/_cppyy/executor.py
+++ b/pypy/module/_cppyy/executor.py
@@ -1,14 +1,10 @@
 import sys
 
 from pypy.interpreter.error import oefmt
-
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib import jit_libffi
-
 from pypy.module._rawffi.interp_rawffi import letter2tp
-from pypy.module._rawffi.array import W_Array, W_ArrayInstance
-
-from pypy.module._cppyy import helper, capi, ffitypes
+from pypy.module._cppyy import helper, capi, ffitypes, lowlevelviews
 
 # Executor objects are used to dispatch C++ methods. They are defined by their
 # return type only: arguments are converted by Converter objects, and Executors
@@ -60,7 +56,7 @@
             from pypy.module._cppyy import interp_cppyy
             return interp_cppyy.get_nullptr(space)
         shape = letter2tp(space, self.typecode)
-        return W_ArrayInstance(space, shape, sys.maxint/shape.size, ptrval)
+        return lowlevelviews.W_LowLevelView(space, shape, sys.maxint/shape.size, ptrval)
 
 
 class VoidExecutor(Executor):
@@ -80,9 +76,6 @@
 class NumericExecutorMixin(object):
     _mixin_ = True
 
-    #def _wrap_object(self, space, obj):
-    #    return getattr(space, self.wrapper)(obj)
-
     def execute(self, space, cppmethod, cppthis, num_args, args):
         result = self.c_stubcall(space, cppmethod, cppthis, num_args, args)
         return self._wrap_object(space, rffi.cast(self.c_type, result))
@@ -98,19 +91,16 @@
     def __init__(self, space, extra):
         Executor.__init__(self, space, extra)
         self.do_assign = False
-        self.item = rffi.cast(self.c_type, 0)
+        self.w_item = space.w_None
 
     def set_item(self, space, w_item):
-        self.item = self._unwrap_object(space, w_item)
+        self.w_item = w_item
         self.do_assign = True
 
-    #def _wrap_object(self, space, obj):
-    #    return getattr(space, self.wrapper)(rffi.cast(self.c_type, obj))
-
     def _wrap_reference(self, space, rffiptr):
         if self.do_assign:
-            rffiptr[0] = self.item
-        self.do_assign = False
+            rffiptr[0] = rffi.cast(self.c_type, self._unwrap_object(space, self.w_item))
+            self.do_assign = False
         return self._wrap_object(space, rffiptr[0])    # all paths, for rtyper
 
     def execute(self, space, cppmethod, cppthis, num_args, args):
@@ -123,6 +113,48 @@
         return self._wrap_reference(space,
             rffi.cast(self.c_ptrtype, rffi.cast(rffi.VOIDPP, result)[0]))
 
+class LongDoubleExecutorMixin(object):
+    # Note: not really supported, but returns normal double
+    _mixin_ = True
+
+    def execute(self, space, cppmethod, cppthis, num_args, args):
+        result = self.c_stubcall(space, cppmethod, cppthis, num_args, args)
+        return space.newfloat(result)
+
+    def execute_libffi(self, space, cif_descr, funcaddr, buffer):
+        from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
+        raise FastCallNotPossible
+
+class LongDoubleExecutor(ffitypes.typeid(rffi.LONGDOUBLE), LongDoubleExecutorMixin, Executor):
+    _immutable_ = True
+    c_stubcall  = staticmethod(capi.c_call_ld)
+
+class LongDoubleRefExecutorMixin(NumericRefExecutorMixin):
+    # Note: not really supported, but returns normal double
+    _mixin_ = True
+
+    def _wrap_reference(self, space, rffiptr):
+        if self.do_assign:
+            capi.c_double2longdouble(space, space.float_w(self.w_item), rffiptr)
+            self.do_assign = False
+            return self.w_item
+        return space.newfloat(capi.c_longdouble2double(space, rffiptr))
+
+    def execute(self, space, cppmethod, cppthis, num_args, args):
+        result = capi.c_call_r(space, cppmethod, cppthis, num_args, args)
+        return self._wrap_reference(space, rffi.cast(self.c_ptrtype, result))
+
+    def execute_libffi(self, space, cif_descr, funcaddr, buffer):
+        jit_libffi.jit_ffi_call(cif_descr, funcaddr, buffer)
+        result = rffi.ptradd(buffer, cif_descr.exchange_result)
+        return self._wrap_reference(space,
+            rffi.cast(self.c_ptrtype, rffi.cast(rffi.VOIDPP, result)[0]))
+
+class LongDoubleRefExecutor(ffitypes.typeid(rffi.LONGDOUBLE), LongDoubleRefExecutorMixin, Executor):
+    def cffi_type(self, space):
+        state = space.fromcache(ffitypes.State)
+        return state.c_voidp
+
 
 class CStringExecutor(Executor):
     def execute(self, space, cppmethod, cppthis, num_args, args):
@@ -349,6 +381,10 @@
 _executors["void*"]               = PtrTypeExecutor
 _executors["const char*"]         = CStringExecutor
 
+# long double not really supported: narrows to double
+_executors["long double"]          = LongDoubleExecutor
+_executors["long double&"]         = LongDoubleRefExecutor
+
 # special cases (note: 'string' aliases added below)
 _executors["constructor"]         = ConstructorExecutor
 
diff --git a/pypy/module/_cppyy/ffitypes.py b/pypy/module/_cppyy/ffitypes.py
--- a/pypy/module/_cppyy/ffitypes.py
+++ b/pypy/module/_cppyy/ffitypes.py
@@ -296,7 +296,8 @@
     _immutable_fields_ = ['c_type', 'c_ptrtype', 'typecode']
 
     c_type      = rffi.LONGDOUBLE
-    c_ptrtype   = rffi.LONGDOUBLEP
+    # c_ptrtype   = rffi.LONGDOUBLEP   # useless type at this point
+    c_ptrtype   = rffi.VOIDP
     typecode    = 'g'
 
     # long double is not really supported ...
@@ -304,7 +305,7 @@
         return r_longfloat(space.float_w(w_obj))
 
     def _wrap_object(self, space, obj):
-        return space.wrap(obj)
+        return space.newfloat(obj)
 
     def cffi_type(self, space):
         state = space.fromcache(State)
diff --git a/pypy/module/_cppyy/helper.py b/pypy/module/_cppyy/helper.py
--- a/pypy/module/_cppyy/helper.py
+++ b/pypy/module/_cppyy/helper.py
@@ -117,16 +117,10 @@
     # TODO: perhaps absorb or "pythonify" these operators?
     return cppname
 
-if sys.hexversion < 0x3000000:
-    CPPYY__div__  = "__div__"
-    CPPYY__idiv__ = "__idiv__"
-    CPPYY__long__ = "__long__"
-    CPPYY__bool__ = "__nonzero__"
-else:
-    CPPYY__div__  = "__truediv__"
-    CPPYY__idiv__ = "__itruediv__"
-    CPPYY__long__ = "__int__"
-    CPPYY__bool__ = "__bool__"
+CPPYY__div__  = "__div__"
+CPPYY__idiv__ = "__idiv__"
+CPPYY__long__ = "__long__"
+CPPYY__bool__ = "__nonzero__"
 
 # _operator_mappings["[]"]  = "__setitem__"      # depends on return type
 # _operator_mappings["+"]   = "__add__"          # depends on # of args (see __pos__)
diff --git a/pypy/module/_cppyy/include/capi.h b/pypy/module/_cppyy/include/capi.h
--- a/pypy/module/_cppyy/include/capi.h
+++ b/pypy/module/_cppyy/include/capi.h
@@ -63,6 +63,8 @@
     double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
     RPY_EXTERN
     long double cppyy_call_ld(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
+    double cppyy_call_nld(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
 
     RPY_EXTERN
     void* cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
@@ -151,11 +153,15 @@
     RPY_EXTERN
     char* cppyy_method_signature(cppyy_method_t, int show_formalargs);
     RPY_EXTERN
-    char* cppyy_method_prototype(cppyy_scope_t scope, cppyy_method_t idx, int show_formalargs);
+    char* cppyy_method_prototype(cppyy_scope_t scope, cppyy_method_t, int show_formalargs);
     RPY_EXTERN
     int cppyy_is_const_method(cppyy_method_t);
 
     RPY_EXTERN
+    int get_num_templated_methods(cppyy_scope_t scope);
+    RPY_EXTERN
+    char* get_templated_method_name(cppyy_scope_t scope, cppyy_index_t imeth);
+    RPY_EXTERN
     int cppyy_exists_method_template(cppyy_scope_t scope, const char* name);
     RPY_EXTERN
     int cppyy_method_is_template(cppyy_scope_t scope, cppyy_index_t idx);
@@ -216,9 +222,14 @@
     cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr);
 
     RPY_EXTERN
-    const char* cppyy_stdvector_valuetype(const char* clname);
+    double cppyy_longdouble2double(void*);
     RPY_EXTERN
-    size_t      cppyy_stdvector_valuesize(const char* clname);
+    void   cppyy_double2longdouble(double, void*);
+
+    RPY_EXTERN
+    int         cppyy_vectorbool_getitem(cppyy_object_t ptr, int idx);
+    RPY_EXTERN
+    void        cppyy_vectorbool_setitem(cppyy_object_t ptr, int idx, int value);
 
 #ifdef __cplusplus
 }
diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -24,6 +24,7 @@
 INSTANCE_FLAGS_IS_RVALUE   = 0x0004
 
 OVERLOAD_FLAGS_USE_FFI     = 0x0001
+OVERLOAD_FLAGS_CREATES     = 0x0002
 
 FUNCTION_IS_GLOBAL         = 0x0001
 FUNCTION_IS_STATIC         = 0x0001
@@ -41,6 +42,7 @@
              'void**'        : 100,
              'float'         :  30,
              'double'        :  10,
+             'bool'          :   1,
              'const string&' :   1, } # solves a specific string ctor overload
 
 from rpython.rlib.listsort import make_timsort_class
@@ -167,6 +169,7 @@
 #
 #  W_CPPOverload:                 instance methods (base class)
 #  W_CPPConstructorOverload:      constructors
+#  W_CPPAbstractCtorOverload:     to provent instantiation of abstract classes
 #  W_CPPStaticOverload:           free and static functions
 #  W_CPPTemplateOverload:         templated methods
 #  W_CPPTemplateStaticOverload:   templated free and static functions
@@ -227,8 +230,10 @@
         if self.converters is None:
             try:
                 self._setup(cppthis)
-            except Exception:
-                pass
+            except Exception as e:
+                if self.converters is None:
+                    raise oefmt(self.space.w_SystemError,
+                        "unable to initialize converters (%s)", str(e))
 
         # attempt to call directly through ffi chain
         if useffi and self._funcaddr:
@@ -258,7 +263,8 @@
         jit.promote(self)
         cif_descr = self.cif_descr
         # add extra space for const-ref support (see converter.py)
-        buffer = lltype.malloc(rffi.CCHARP.TO, cif_descr.exchange_size+len(self.arg_defs)*rffi.sizeof(rffi.DOUBLE), flavor='raw')
+        buffer = lltype.malloc(rffi.CCHARP.TO,
+            cif_descr.exchange_size+len(self.arg_defs)*rffi.sizeof(rffi.DOUBLE), flavor='raw')
         thisoff = 0
         try:
             if cppthis:
@@ -412,8 +418,10 @@
 
     def priority(self):
         total_arg_priority = 0
-        for p in [priority.get(arg_type, 0) for arg_type, arg_dflt in self.arg_defs]:
-            total_arg_priority += p
+        for arg_type, arg_dflt in self.arg_defs:
+            total_arg_priority += priority.get(arg_type, 0)
+            if '&&' in arg_type:
+                total_arg_priority += 100
         return total_arg_priority
 
     @rgc.must_be_light_finalizer
@@ -433,7 +441,7 @@
 
 class CPPSetItem(CPPMethod):
     """Method dispatcher specific to Python's __setitem__ mapped onto C++'s
-    operator[](int). The former function takes an extra argument to assign to
+    operator[](T). The former function takes an extra argument to assign to
     the return type of the latter."""
 
     _attrs_ = []
@@ -453,6 +461,36 @@
 # need forwarding, which the normal instancemethod does not provide, hence this
 # derived class.
 class MethodWithProps(Method):
+    # set life management of result from the call
+    def fget_creates(self, space):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        return f.fget_creates(space)
+
+    @unwrap_spec(value=bool)
+    def fset_creates(self, space, value):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        f.fset_creates(space, value)
+
+    # set ownership policy of arguments (not yet implemented)
+    def fget_mempolicy(self, space):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        return f.fget_mempolicy(space)
+
+    @unwrap_spec(value=int)
+    def fset_mempolicy(self, space, value):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        f.fset_mempolicy(space, value)
+
+    # set to release the gil during call (not yet implemented)
+    def fget_release_gil(self, space):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        return f.fget_release_gil(space)
+
+    @unwrap_spec(value=bool)
+    def fset_release_gil(self, space, value):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        f.fset_release_gil(space, value)
+
     # allow user to determine ffi use rules per overload
     def fget_useffi(self, space):
         f = space.interp_w(W_CPPOverload, self.w_function)
@@ -468,19 +506,22 @@
     __doc__ = """cpp_instancemethod(function, instance, class)
 
 Create an instance method object.""",
-    __new__ = interp2app(MethodWithProps.descr_method__new__.im_func),
-    __call__ = interp2app(MethodWithProps.descr_method_call),
-    __get__ = interp2app(MethodWithProps.descr_method_get),
-    __func__ = interp_attrproperty_w('w_function', cls=MethodWithProps),
-    __self__ = interp_attrproperty_w('w_instance', cls=MethodWithProps),
+    __new__          = interp2app(MethodWithProps.descr_method__new__.im_func),
+    __call__         = interp2app(MethodWithProps.descr_method_call),
+    __get__          = interp2app(MethodWithProps.descr_method_get),
+    __func__         = interp_attrproperty_w('w_function', cls=MethodWithProps),
+    __self__         = interp_attrproperty_w('w_instance', cls=MethodWithProps),
     __getattribute__ = interp2app(MethodWithProps.descr_method_getattribute),
-    __eq__ = interp2app(MethodWithProps.descr_method_eq),
-    __ne__ = descr_generic_ne,
-    __hash__ = interp2app(MethodWithProps.descr_method_hash),
-    __repr__ = interp2app(MethodWithProps.descr_method_repr),
-    __reduce__ = interp2app(MethodWithProps.descr_method__reduce__),
-    __weakref__ = make_weakref_descr(MethodWithProps),
-    __useffi__ = GetSetProperty(MethodWithProps.fget_useffi, MethodWithProps.fset_useffi),
+    __eq__           = interp2app(MethodWithProps.descr_method_eq),
+    __ne__           = descr_generic_ne,
+    __hash__         = interp2app(MethodWithProps.descr_method_hash),
+    __repr__         = interp2app(MethodWithProps.descr_method_repr),
+    __reduce__       = interp2app(MethodWithProps.descr_method__reduce__),
+    __weakref__      = make_weakref_descr(MethodWithProps),
+    __creates__      = GetSetProperty(MethodWithProps.fget_creates,     MethodWithProps.fset_creates),
+    __mempolicy__    = GetSetProperty(MethodWithProps.fget_mempolicy,   MethodWithProps.fset_mempolicy),
+    __release_gil__  = GetSetProperty(MethodWithProps.fget_release_gil, MethodWithProps.fset_release_gil),
+    __useffi__       = GetSetProperty(MethodWithProps.fget_useffi,      MethodWithProps.fset_useffi),
     )
 MethodWithProps.typedef.acceptable_as_base_class = False
 
@@ -502,6 +543,9 @@
     def descr_get(self, w_obj, w_cls=None):
         """functionobject.__get__(obj[, type]) -> method"""
         # TODO: check validity of w_cls if given
+        # TODO: this does not work for Python 3, which does not have
+        #  unbound methods (probably no common code possible, see also
+        #  pypy/interpreter/function.py)
         space = self.space
         asking_for_bound = (space.is_none(w_cls) or
                             not space.is_w(w_obj, space.w_None) or
@@ -540,7 +584,12 @@
         for i in range(len(self.functions)):
             cppyyfunc = self.functions[i]
             try:
-                return cppyyfunc.call(cppthis, args_w, self.flags & OVERLOAD_FLAGS_USE_FFI)
+                w_result = cppyyfunc.call(cppthis, args_w, self.flags & OVERLOAD_FLAGS_USE_FFI)
+                if self.flags & OVERLOAD_FLAGS_CREATES:
+                    if isinstance(w_result, W_CPPInstance):
+                        cppinstance = self.space.interp_w(W_CPPInstance, w_result)
+                        cppinstance.fset_python_owns(self.space, self.space.w_True)
+                return w_result
             except Exception:
                 pass
 
@@ -553,6 +602,7 @@
         for i in range(len(self.functions)):
             cppyyfunc = self.functions[i]
             try:
+                # no need to set ownership on the return value, as none of the methods execute
                 return cppyyfunc.call(cppthis, args_w, self.flags & OVERLOAD_FLAGS_USE_FFI)
             except OperationError as e:
                 # special case if there's just one function, to prevent clogging the error message
@@ -581,6 +631,43 @@
             sig += '\n'+self.functions[i].prototype()
         return self.space.newtext(sig)
 
+    @unwrap_spec(signature='text')
+    def mp_overload(self, signature):
+        sig = '(%s)' % signature
+        for f in self.functions:
+            if f.signature(False) == sig:
+                if isinstance(self, W_CPPStaticOverload):
+                    return W_CPPStaticOverload(self.space, self.scope, [f])
+                return W_CPPOverload(self.space, self.scope, [f])
+        raise oefmt(self.space.w_LookupError, "signature '%s' not found", signature)
+
+    # set life management of result from the call
+    def fget_creates(self, space):
+        return space.newbool(bool(self.flags & OVERLOAD_FLAGS_CREATES))
+
+    @unwrap_spec(value=bool)
+    def fset_creates(self, space, value):
+        if space.is_true(value):
+            self.flags |= OVERLOAD_FLAGS_CREATES
+        else:
+            self.flags &= ~OVERLOAD_FLAGS_CREATES
+
+    # set ownership policy of arguments (not yet implemented)
+    def fget_mempolicy(self, space):
+        return space.newint(0)
+
+    @unwrap_spec(value=int)
+    def fset_mempolicy(self, space, value):
+        pass
+
+    # set to release the gil during call (not yet implemented)
+    def fget_release_gil(self, space):
+        return space.newbool(True)
+
+    @unwrap_spec(value=bool)
+    def fset_release_gil(self, space, value):
+        pass
+
     # allow user to determine ffi use rules per overload
     def fget_useffi(self, space):
         return space.newbool(bool(self.flags & OVERLOAD_FLAGS_USE_FFI))
@@ -604,10 +691,14 @@
 
 W_CPPOverload.typedef = TypeDef(
     'CPPOverload',
-    __get__    = interp2app(W_CPPOverload.descr_get),
-    __call__   = interp2app(W_CPPOverload.call_args),
-    __useffi__ = GetSetProperty(W_CPPOverload.fget_useffi, W_CPPOverload.fset_useffi),
-    __doc__    = GetSetProperty(W_CPPOverload.fget_doc)
+    __get__         = interp2app(W_CPPOverload.descr_get),
+    __call__        = interp2app(W_CPPOverload.call_args),
+    __creates__     = GetSetProperty(W_CPPOverload.fget_creates,     W_CPPOverload.fset_creates),
+    __mempolicy__   = GetSetProperty(W_CPPOverload.fget_mempolicy,   W_CPPOverload.fset_mempolicy),
+    __release_gil__ = GetSetProperty(W_CPPOverload.fget_release_gil, W_CPPOverload.fset_release_gil),
+    __useffi__      = GetSetProperty(W_CPPOverload.fget_useffi,      W_CPPOverload.fset_useffi),
+    __overload__    = interp2app(W_CPPOverload.mp_overload),
+    __doc__         = GetSetProperty(W_CPPOverload.fget_doc)
 )
 
 
@@ -629,21 +720,21 @@
     @unwrap_spec(args_w='args_w')
     def call_args(self, args_w):
         jit.promote(self)
-        #if isinstance(args_w[0], W_CPPInstance):
-            # free function used as bound method, leave in place
         return self.call_impl(capi.C_NULL_OBJECT, args_w)
-        # free functions are implemented as methods of 'namespace' classes, remove 'instance'
-        #return self.call_impl(capi.C_NULL_OBJECT, args_w[1:])
 
     def __repr__(self):
         return "W_CPPStaticOverload(%s)" % [f.prototype() for f in self.functions]
 
 W_CPPStaticOverload.typedef = TypeDef(
     'CPPStaticOverload',
-    __get__    = interp2app(W_CPPStaticOverload.descr_get),
-    __call__   = interp2app(W_CPPStaticOverload.call_args),
-    __useffi__ = GetSetProperty(W_CPPStaticOverload.fget_useffi, W_CPPStaticOverload.fset_useffi),
-    __doc__    = GetSetProperty(W_CPPStaticOverload.fget_doc)
+    __get__         = interp2app(W_CPPStaticOverload.descr_get),
+    __call__        = interp2app(W_CPPStaticOverload.call_args),
+    __creates__     = GetSetProperty(W_CPPStaticOverload.fget_creates,     W_CPPStaticOverload.fset_creates),
+    __mempolicy__   = GetSetProperty(W_CPPStaticOverload.fget_mempolicy,   W_CPPStaticOverload.fset_mempolicy),
+    __release_gil__ = GetSetProperty(W_CPPStaticOverload.fget_release_gil, W_CPPStaticOverload.fset_release_gil),
+    __useffi__      = GetSetProperty(W_CPPStaticOverload.fget_useffi,      W_CPPStaticOverload.fset_useffi),
+    __overload__    = interp2app(W_CPPStaticOverload.mp_overload),
+    __doc__         = GetSetProperty(W_CPPStaticOverload.fget_doc)
 )
 
 
@@ -657,11 +748,6 @@
     @unwrap_spec(args_w='args_w')
     def call_args(self, args_w):
         jit.promote(self)
-        # TODO: factor out the following:
-        if capi.c_is_abstract(self.space, self.scope.handle):
-            raise oefmt(self.space.w_TypeError,
-                        "cannot instantiate abstract class '%s'",
-                        self.scope.name)
         cppinstance = self.space.interp_w(W_CPPInstance, args_w[0])
         w_result = self.call_impl(rffi.cast(capi.C_OBJECT, self.scope.handle), args_w[1:])
         newthis = rffi.cast(capi.C_OBJECT, self.space.uint_w(w_result))
@@ -674,15 +760,34 @@
 
 W_CPPConstructorOverload.typedef = TypeDef(
     'CPPConstructorOverload',
-    __get__    = interp2app(W_CPPConstructorOverload.descr_get),
-    __call__   = interp2app(W_CPPConstructorOverload.call_args),
-    __doc__    = GetSetProperty(W_CPPConstructorOverload.fget_doc)
+    __get__      = interp2app(W_CPPConstructorOverload.descr_get),
+    __call__     = interp2app(W_CPPConstructorOverload.call_args),
+    __overload__ = interp2app(W_CPPConstructorOverload.mp_overload),
+    __doc__      = GetSetProperty(W_CPPConstructorOverload.fget_doc)
+)
+
+class W_CPPAbstractCtorOverload(W_CPPOverload):
+    _attrs_ = []
+
+    @unwrap_spec(args_w='args_w')
+    def call_args(self, args_w):
+        raise oefmt(self.space.w_TypeError,
+            "cannot instantiate abstract class '%s'", self.scope.name)
+
+    def __repr__(self):
+        return "W_CPPAbstractCtorOverload"
+
+W_CPPAbstractCtorOverload.typedef = TypeDef(
+    'CPPAbstractCtorOverload',
+    __get__    = interp2app(W_CPPAbstractCtorOverload.descr_get),
+    __call__   = interp2app(W_CPPAbstractCtorOverload.call_args),
 )
 
 
 class TemplateOverloadMixin(object):
     """Mixin to instantiate templated methods/functions."""
 
+    _attrs_ = ['tmpl_args_w']
     _mixin_ = True
 
     def construct_template_args(self, w_tpArgs, args_w = None):
@@ -699,7 +804,7 @@
                     if args_w:
                         # try to specialize the type match for the given object
                         cppinstance = self.space.interp_w(W_CPPInstance, args_w[i])
-                        if cppinstance.flags & INSTANCE_FLAGS_IS_RVALUE:
+                        if cppinstance.rt_flags & INSTANCE_FLAGS_IS_RVALUE:
                             sugar = "&&"
                         elif cppinstance.flags & INSTANCE_FLAGS_IS_REF:
                             sugar = "*"
@@ -735,23 +840,37 @@
         return cppol
 
     def instantiate_and_call(self, name, args_w):
-        # try to match with run-time instantiations
-        for cppol in self.master.overloads.values():
-            try:
-                if not self.space.is_w(self.w_this, self.space.w_None):
-                    return self.space.call_obj_args(cppol, self.w_this, Arguments(self.space, args_w))
-                return self.space.call_args(cppol, Arguments(self.space, args_w))
-            except Exception:
-                pass    # completely ignore for now; have to see whether errors become confusing
+        method = None
+        try:
+            # existing cached instantiations
+            if name[-1] == '>':   # only accept full templated name, to ensure explicit
+                method = self.master.overloads[name]
+            else:
+            # try to match with run-time instantiations
+                # TODO: logically, this could be used, but in practice, it's proving too
+                #  greedy ... maybe as a last resort?
+                #for cppol in self.master.overloads.values():
+                #    try:
+                #        if not self.space.is_w(self.w_this, self.space.w_None):
+                #            return self.space.call_obj_args(cppol, self.w_this, Arguments(self.space, args_w))
+                #        return self.space.call_args(cppol, Arguments(self.space, args_w))
+                #    except Exception:
+                #        pass    # completely ignore for now; have to see whether errors become confusing
+                raise TypeError("pre-existing overloads failed")
+        except (KeyError, TypeError):
+            # if not known, try to deduce from argument types
+            w_types = self.space.newtuple([self.space.type(obj_w) for obj_w in args_w])
+            proto = self.construct_template_args(w_types, args_w)
+            method = self.find_method_template(name, proto)
 
-        # if all failed, then try to deduce from argument types
-        w_types = self.space.newtuple([self.space.type(obj_w) for obj_w in args_w])
-        proto = self.construct_template_args(w_types, args_w)
-        method = self.find_method_template(name, proto)
-
-        # only cache result if the name retains the full template
-        fullname = capi.c_method_full_name(self.space, method.functions[0].cppmethod)
-        if 0 <= fullname.rfind('>'):
+            # only cache result if the name retains the full template
+            # TODO: the problem is in part that c_method_full_name returns incorrect names,
+            # e.g. when default template arguments are involved, so for now use 'name' if it
+            # has the full templated name
+            if name[-1] == '>':
+                fullname = name
+            else:
+                fullname = capi.c_method_full_name(self.space, method.functions[0].cppmethod)
             try:
                 existing = self.master.overloads[fullname]
                 allf = existing.functions + method.functions
@@ -763,9 +882,10 @@
             except KeyError:
                 self.master.overloads[fullname] = method
 
-        if not self.space.is_w(self.w_this, self.space.w_None):
-            return self.space.call_obj_args(method, self.w_this, Arguments(self.space, args_w))
-        return self.space.call_args(method, Arguments(self.space, args_w))
+        if method is not None:
+            if not self.space.is_w(self.w_this, self.space.w_None):
+                return self.space.call_obj_args(method, self.w_this, Arguments(self.space, args_w))
+            return self.space.call_args(method, Arguments(self.space, args_w))
 
     def getitem_impl(self, name, args_w):
         space = self.space
@@ -779,14 +899,9 @@
         fullname = name+'<'+tmpl_args+'>'
         try:
             method = self.master.overloads[fullname]
-        except KeyError:
-            method = self.find_method_template(fullname)
-            # cache result (name is always full templated name)
-            self.master.overloads[fullname] = method
-            # also cache on "official" name (may include default template arguments)
-            c_fullname = capi.c_method_full_name(self.space, method.functions[0].cppmethod)
-            if c_fullname != fullname:
-                self.master.overloads[c_fullname] = method
+        except KeyError as e:
+            # defer instantiation until arguments are known
+            return self.clone(tmpl_args)
 
         return method.descr_get(self.w_this, None)
 
@@ -795,21 +910,29 @@
     """App-level dispatcher to allow both lookup/instantiation of templated methods and
     dispatch among overloads between templated and non-templated method."""
 
-    _attrs_ = ['name', 'overloads', 'master', 'w_this']
-    _immutable_fields_ = ['name']
+    _attrs_ = ['name', 'tmpl_args', 'overloads', 'master', 'w_this']
+    _immutable_fields_ = ['name', 'tmpl_args']
 
-    def __init__(self, space, name, decl_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
+    def __init__(self, space, name, tmpl_args, decl_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
          W_CPPOverload.__init__(self, space, decl_scope, functions, flags)
          self.name = name
+         self.tmpl_args = tmpl_args
          self.overloads = {}
          self.master = self
          self.w_this = space.w_None
 
+    def clone(self, tmpl_args):
+        other = W_CPPTemplateOverload(self.space, self.name, tmpl_args, self.scope, self.functions, self.flags)
+        other.overloads = self.overloads
+        other.master = self.master
+        other.w_this = self.w_this
+        return other
+
     def descr_get(self, w_cppinstance, w_cls=None):
         # TODO: don't return copy, but bind in an external object (like W_CPPOverload)
         if self.space.is_w(w_cppinstance, self.space.w_None):
             return self  # unbound, so no new instance needed
-        cppol = W_CPPTemplateOverload(self.space, self.name, self.scope, self.functions, self.flags)
+        cppol = W_CPPTemplateOverload(self.space, self.name, self.tmpl_args, self.scope, self.functions, self.flags)
         cppol.w_this = w_cppinstance
         cppol.master = self.master
         return cppol     # bound
@@ -819,13 +942,18 @@
         # direct call: either pick non-templated overload or attempt to deduce
         # the template instantiation from the argument types
 
-        # try existing overloads or compile-time instantiations
+        # do explicit lookup with tmpl_args if given
         try:
-            return W_CPPOverload.call_args(self, args_w)
+            fullname = self.name
+            if self.tmpl_args is not None:
+                fullname = fullname+'<'+self.tmpl_args+'>'
+            return self.instantiate_and_call(fullname, args_w)
         except Exception:
             pass
 
-        return self.instantiate_and_call(self.name, args_w)
+        # otherwise, try existing overloads or compile-time instantiations
+        # TODO: consolidate errors
+        return W_CPPOverload.call_args(self, [self.w_this]+args_w)
 
     @unwrap_spec(args_w='args_w')
     def getitem(self, args_w):
@@ -839,33 +967,44 @@
 
 W_CPPTemplateOverload.typedef = TypeDef(
     'CPPTemplateOverload',
-    __get__     = interp2app(W_CPPTemplateOverload.descr_get),
-    __getitem__ = interp2app(W_CPPTemplateOverload.getitem),
-    __call__    = interp2app(W_CPPTemplateOverload.call_args),
-    __useffi__  = GetSetProperty(W_CPPTemplateOverload.fget_useffi, W_CPPTemplateOverload.fset_useffi),
-    __doc__     = GetSetProperty(W_CPPTemplateOverload.fget_doc)
+    __get__         = interp2app(W_CPPTemplateOverload.descr_get),
+    __getitem__     = interp2app(W_CPPTemplateOverload.getitem),
+    __call__        = interp2app(W_CPPTemplateOverload.call_args),
+    __creates__     = GetSetProperty(W_CPPTemplateOverload.fget_creates,     W_CPPTemplateOverload.fset_creates),
+    __mempolicy__   = GetSetProperty(W_CPPTemplateOverload.fget_mempolicy,   W_CPPTemplateOverload.fset_mempolicy),
+    __release_gil__ = GetSetProperty(W_CPPTemplateOverload.fget_release_gil, W_CPPTemplateOverload.fset_release_gil),
+    __useffi__      = GetSetProperty(W_CPPTemplateOverload.fget_useffi,      W_CPPTemplateOverload.fset_useffi),
+    __doc__         = GetSetProperty(W_CPPTemplateOverload.fget_doc)
 )
 
 class W_CPPTemplateStaticOverload(W_CPPStaticOverload, TemplateOverloadMixin):
     """App-level dispatcher to allow both lookup/instantiation of templated methods and
     dispatch among overloads between templated and non-templated method."""
 
-    _attrs_ = ['name', 'overloads', 'master', 'w_this']
-    _immutable_fields_ = ['name']
+    _attrs_ = ['name', 'tmpl_args', 'overloads', 'master', 'w_this']
+    _immutable_fields_ = ['name', 'tmpl_args']
 
-    def __init__(self, space, name, decl_scope, funcs, flags = OVERLOAD_FLAGS_USE_FFI):
+    def __init__(self, space, name, tmpl_args, decl_scope, funcs, flags = OVERLOAD_FLAGS_USE_FFI):
          W_CPPStaticOverload.__init__(self, space, decl_scope, funcs, flags)
          self.name = name
+         self.tmpl_args = tmpl_args
          self.overloads = {}
          self.master = self
          self.w_this = space.w_None
 
+    def clone(self, tmpl_args):
+        other = W_CPPTemplateStaticOverload(self.space, self.name, tmpl_args, self.scope, self.functions, self.flags)
+        other.overloads = self.overloads
+        other.master = self.master
+        other.w_this = self.w_this
+        return other
+
     def descr_get(self, w_cppinstance, w_cls=None):
         # TODO: don't return copy, but bind in an external object (like W_CPPOverload)
         if isinstance(w_cppinstance, W_CPPInstance):
             cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance)
             if cppinstance.clsdecl.handle != self.scope.handle:
-                cppol = W_CPPTemplateStaticOverload(self.space, self.name, self.scope, self.functions, self.flags)
+                cppol = W_CPPTemplateStaticOverload(self.space, self.name, self.tmpl_args, self.scope, self.functions, self.flags)
                 cppol.w_this = w_cppinstance
                 cppol.master = self.master
                 return cppol       # bound
@@ -875,15 +1014,20 @@
     def call_args(self, args_w):
         # direct call: either pick non-templated overload or attempt to deduce
         # the template instantiation from the argument types
+        # TODO: refactor with W_CPPTemplateOverload
 
-        # try existing overloads or compile-time instantiations
+        # do explicit lookup with tmpl_args if given
         try:
-            return W_CPPStaticOverload.call_args(self, args_w)
+            fullname = self.name
+            if self.tmpl_args is not None:
+                fullname = fullname+'<'+self.tmpl_args+'>'
+            return self.instantiate_and_call(fullname, args_w)
         except Exception:
             pass
 
-        # try new instantiation
-        return self.instantiate_and_call(self.name, args_w)
+        # otherwise, try existing overloads or compile-time instantiations
+        # TODO: consolidate errors
+        return W_CPPStaticOverload.call_args(self, args_w)
 
     @unwrap_spec(args_w='args_w')
     def getitem(self, args_w):
@@ -897,11 +1041,18 @@
 
 W_CPPTemplateStaticOverload.typedef = TypeDef(
     'CPPTemplateStaticOverload',
-    __get__     = interp2app(W_CPPTemplateStaticOverload.descr_get),
-    __getitem__ = interp2app(W_CPPTemplateStaticOverload.getitem),
-    __call__    = interp2app(W_CPPTemplateStaticOverload.call_args),
-    __useffi__  = GetSetProperty(W_CPPTemplateStaticOverload.fget_useffi, W_CPPTemplateStaticOverload.fset_useffi),
-    __doc__     = GetSetProperty(W_CPPTemplateStaticOverload.fget_doc)
+    __get__         = interp2app(W_CPPTemplateStaticOverload.descr_get),
+    __getitem__     = interp2app(W_CPPTemplateStaticOverload.getitem),
+    __call__        = interp2app(W_CPPTemplateStaticOverload.call_args),
+    __creates__     = GetSetProperty(W_CPPTemplateStaticOverload.fget_creates,
+                                     W_CPPTemplateStaticOverload.fset_creates),
+    __mempolicy__   = GetSetProperty(W_CPPTemplateStaticOverload.fget_mempolicy,
+                                     W_CPPTemplateStaticOverload.fset_mempolicy),
+    __release_gil__ = GetSetProperty(W_CPPTemplateStaticOverload.fget_release_gil,
+                                     W_CPPTemplateStaticOverload.fset_release_gil),
+    __useffi__      = GetSetProperty(W_CPPTemplateStaticOverload.fget_useffi,

From pypy.commits at gmail.com  Mon Aug 13 22:39:23 2018
From: pypy.commits at gmail.com (mattip)
Date: Mon, 13 Aug 2018 19:39:23 -0700 (PDT)
Subject: [pypy-commit] cffi doc: edit overview
Message-ID: <5b7240db.1c69fb81.2df5.65da@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: doc
Changeset: r3139:bb2d9d0e810b
Date: 2018-08-13 19:38 -0700
http://bitbucket.org/cffi/cffi/changeset/bb2d9d0e810b/

Log:	edit overview

diff --git a/doc/source/overview.rst b/doc/source/overview.rst
--- a/doc/source/overview.rst
+++ b/doc/source/overview.rst
@@ -5,10 +5,11 @@
 .. contents::
    
 
-This document starts, in the first section, with a simple working
-example of using CFFI to call a C function from Python. CFFI is
+The first section presents a simple working
+example of using CFFI to call a C function in a compiled shared objecti
+from Python. CFFI is
 flexible and covers several other use cases presented in the second
-section. Then, the next section shows how to export Python functions
+section. The third section shows how to export Python functions
 to a Python interpreter embedded in a C or C++ application. The last
 two sections delve deeper in the CFFI library.
 
@@ -24,12 +25,13 @@
 ------------------
 
 The main way to use CFFI is as an interface to some already-compiled
-library which is provided by other means.  Imagine that you have a
-system-installed library called ``piapprox.dll`` (Windows) or
+shared object which is provided by other means.  Imagine that you have a
+system-installed shared object called ``piapprox.dll`` (Windows) or
 ``libpiapprox.so`` (Linux and others) or ``libpiapprox.dylib`` (OS X),
-containing a function ``float pi_approx(int n);`` that computes some
-approximation of pi given a number of iterations.  You want to call
-this function from Python.
+exporting a function ``float pi_approx(int n);`` that computes some
+approximation of pi given a number of iterations. You want to call
+this function from Python. Note this method works equally as well with a
+library ``piapprox.lib`` (Windows) or ``libpiapprox.a``.
 
 Create the file ``piapprox_build.py``:
 
@@ -38,13 +40,15 @@
       from cffi import FFI
       ffibuilder = FFI()
 
-      # cdef() expects a string listing the C types, functions and
-      # globals needed from Python. The string follows the C syntax.
+      # cdef() expects a string declaring the C types, functions and
+      # globals needed to use the shared object. It must be in valid C syntax.
       ffibuilder.cdef("""
           float pi_approx(int n);
       """)
 
-      # This describes the extension module "_pi_cffi" to produce.
+      # set_source() names the python extension module "_pi_cffi" to produce
+      # and the C source code as a string that once compiled will call the
+      # function.
       ffibuilder.set_source("_pi_cffi",
       """
       	   #include "pi.h"   // the C header of the library
@@ -57,7 +61,7 @@
 Execute this script.  If everything is OK, it should produce
 ``_pi_cffi.c``, and then invoke the compiler on it.  The produced
 ``_pi_cffi.c`` contains a copy of the string given in ``set_source()``,
-in this example the ``#include "pi.h"``; then it contains some glue code
+in this example the ``#include "pi.h"``. It also contains some glue code
 for all the functions declared in the ``cdef()`` above.
 
 At runtime, you use the extension module like this:
@@ -113,6 +117,8 @@
 Simple example (ABI level, in-line)
 +++++++++++++++++++++++++++++++++++
 
+May look familiar to those who have used ctypes_.
+
 .. code-block:: python
 
     >>> from cffi import FFI
@@ -121,15 +127,14 @@
     ...     int printf(const char *format, ...);   // copy-pasted from the man page
     ... """)                                  
     >>> C = ffi.dlopen(None)                     # loads the entire C namespace
-    >>> arg = ffi.new("char[]", "world")         # equivalent to C code: char arg[] = "world";
-    >>> C.printf("hi there, %s.\n", arg)         # call printf
+    >>> arg = ffi.new("char[]", b"world")         # equivalent to C code: char arg[] = "world";
+    >>> C.printf(b"hi there, %s.\n", arg)         # call printf
     hi there, world.
     17                                           # this is the return value
     >>>
 
-Note that on Python 3 you need to pass byte strings to ``char *``
-arguments.  In the above example it would be ``b"world"`` and ``b"hi
-there, %s!\n"``.  In general it is ``somestring.encode(myencoding)``.
+In general you would encode python ``str`` to ``bytes`` with
+``somestring.encode(myencoding)`` on Python3.
 
 *Python 3 on Windows:* ``ffi.dlopen(None)`` does not work.  This problem
 is messy and not really fixable.  The problem does not occur if you try
@@ -172,11 +177,12 @@
     f.close()
 
 This can be used as a more flexible replacement of the struct_ and
-array_ modules.  You could also call ``ffi.new("pixel_t[600][800]")``
+array_ modules, and replaces ctypes_.  You could also call ``ffi.new("pixel_t[600][800]")``
 and get a two-dimensional array.
 
 .. _struct: http://docs.python.org/library/struct.html
 .. _array: http://docs.python.org/library/array.html
+.. _ctypes: http://docs.python.org/library/ctypes.html
 
 *This example does not call any C compiler.*
 
@@ -430,6 +436,7 @@
     from cffi import FFI
 
     ffibuilder = FFI()
+    # Note that the actual source is None
     ffibuilder.set_source("_simple_example", None)
     ffibuilder.cdef("""
         int printf(const char *format, ...);
@@ -478,13 +485,17 @@
     )
 
 
+In summary, this mode is useful when you wish to declare many C structures but
+do not need fast interaction with a shared object. It is useful for parsing
+binary files, for instance.
+
 In-line, API level
 ++++++++++++++++++
 
 The "API level + in-line" mode combination exists but is long
 deprecated.  It used to be done with ``lib = ffi.verify("C header")``.
 The out-of-line variant with ``set_source("modname", "C header")`` is
-preferred.  It avoids a number of problems when the project grows in
+preferred and avoids a number of problems when the project grows in
 size.
 
 
@@ -589,8 +600,8 @@
 needs to go through the very general *libffi* library, which is slow
 (and not always perfectly tested on non-standard platforms).  The API
 mode instead compiles a CPython C wrapper that directly invokes the
-target function.  It is, comparatively, massively faster (and works
-better than libffi ever can).
+target function.  It can be massively faster (and works
+better than libffi ever will).
 
 The more fundamental reason to prefer the API mode is that *the C
 libraries are typically meant to be used with a C compiler.* You are not

From pypy.commits at gmail.com  Tue Aug 14 03:11:14 2018
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Aug 2018 00:11:14 -0700 (PDT)
Subject: [pypy-commit] cffi doc: Small tweaks
Message-ID: <5b728092.1c69fb81.17c02.fc02@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: doc
Changeset: r3140:2c08c41b6523
Date: 2018-08-14 09:10 +0200
http://bitbucket.org/cffi/cffi/changeset/2c08c41b6523/

Log:	Small tweaks

diff --git a/doc/source/overview.rst b/doc/source/overview.rst
--- a/doc/source/overview.rst
+++ b/doc/source/overview.rst
@@ -6,8 +6,8 @@
    
 
 The first section presents a simple working
-example of using CFFI to call a C function in a compiled shared objecti
-from Python. CFFI is
+example of using CFFI to call a C function in a compiled shared object
+(DLL) from Python. CFFI is
 flexible and covers several other use cases presented in the second
 section. The third section shows how to export Python functions
 to a Python interpreter embedded in a C or C++ application. The last
@@ -30,8 +30,8 @@
 ``libpiapprox.so`` (Linux and others) or ``libpiapprox.dylib`` (OS X),
 exporting a function ``float pi_approx(int n);`` that computes some
 approximation of pi given a number of iterations. You want to call
-this function from Python. Note this method works equally as well with a
-library ``piapprox.lib`` (Windows) or ``libpiapprox.a``.
+this function from Python. Note this method works equally well with a
+static library ``piapprox.lib`` (Windows) or ``libpiapprox.a``.
 
 Create the file ``piapprox_build.py``:
 
@@ -40,15 +40,16 @@
       from cffi import FFI
       ffibuilder = FFI()
 
-      # cdef() expects a string declaring the C types, functions and
+      # cdef() expects a single string declaring the C types, functions and
       # globals needed to use the shared object. It must be in valid C syntax.
       ffibuilder.cdef("""
           float pi_approx(int n);
       """)
 
-      # set_source() names the python extension module "_pi_cffi" to produce
-      # and the C source code as a string that once compiled will call the
-      # function.
+      # set_source() gives the name of the python extension module to
+      # produce, and some C source code as a string.  This C code needs
+      # to make the declarated functions, types and globals available,
+      # so it is often just the "#include".
       ffibuilder.set_source("_pi_cffi",
       """
       	   #include "pi.h"   // the C header of the library
@@ -61,8 +62,8 @@
 Execute this script.  If everything is OK, it should produce
 ``_pi_cffi.c``, and then invoke the compiler on it.  The produced
 ``_pi_cffi.c`` contains a copy of the string given in ``set_source()``,
-in this example the ``#include "pi.h"``. It also contains some glue code
-for all the functions declared in the ``cdef()`` above.
+in this example the ``#include "pi.h"``. Afterwards, it contains glue code
+for all the functions, types and globals declared in the ``cdef()`` above.
 
 At runtime, you use the extension module like this:
 
@@ -127,14 +128,15 @@
     ...     int printf(const char *format, ...);   // copy-pasted from the man page
     ... """)                                  
     >>> C = ffi.dlopen(None)                     # loads the entire C namespace
-    >>> arg = ffi.new("char[]", b"world")         # equivalent to C code: char arg[] = "world";
-    >>> C.printf(b"hi there, %s.\n", arg)         # call printf
+    >>> arg = ffi.new("char[]", b"world")        # equivalent to C code: char arg[] = "world";
+    >>> C.printf(b"hi there, %s.\n", arg)        # call printf
     hi there, world.
     17                                           # this is the return value
     >>>
 
-In general you would encode python ``str`` to ``bytes`` with
-``somestring.encode(myencoding)`` on Python3.
+Note that ``char *`` arguments expect a ``bytes`` object.  If you have a
+``str`` (or a ``unicode`` on Python 2) you need to encode it explicitly
+with ``somestring.encode(myencoding)``.
 
 *Python 3 on Windows:* ``ffi.dlopen(None)`` does not work.  This problem
 is messy and not really fixable.  The problem does not occur if you try
@@ -484,11 +486,11 @@
         install_requires=["cffi>=1.0.0"],
     )
 
-
 In summary, this mode is useful when you wish to declare many C structures but
 do not need fast interaction with a shared object. It is useful for parsing
 binary files, for instance.
 
+
 In-line, API level
 ++++++++++++++++++
 

From pypy.commits at gmail.com  Tue Aug 14 03:11:16 2018
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Aug 2018 00:11:16 -0700 (PDT)
Subject: [pypy-commit] cffi doc: ready to merge
Message-ID: <5b728094.1c69fb81.35672.5921@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: doc
Changeset: r3141:6fa8df02b27f
Date: 2018-08-14 09:10 +0200
http://bitbucket.org/cffi/cffi/changeset/6fa8df02b27f/

Log:	ready to merge


From pypy.commits at gmail.com  Tue Aug 14 03:11:18 2018
From: pypy.commits at gmail.com (arigo)
Date: Tue, 14 Aug 2018 00:11:18 -0700 (PDT)
Subject: [pypy-commit] cffi default: hg merge doc
Message-ID: <5b728096.1c69fb81.2d72.5116@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3142:412517069d05
Date: 2018-08-14 09:11 +0200
http://bitbucket.org/cffi/cffi/changeset/412517069d05/

Log:	hg merge doc

diff --git a/doc/source/overview.rst b/doc/source/overview.rst
--- a/doc/source/overview.rst
+++ b/doc/source/overview.rst
@@ -5,10 +5,11 @@
 .. contents::
    
 
-This document starts, in the first section, with a simple working
-example of using CFFI to call a C function from Python. CFFI is
+The first section presents a simple working
+example of using CFFI to call a C function in a compiled shared object
+(DLL) from Python. CFFI is
 flexible and covers several other use cases presented in the second
-section. Then, the next section shows how to export Python functions
+section. The third section shows how to export Python functions
 to a Python interpreter embedded in a C or C++ application. The last
 two sections delve deeper in the CFFI library.
 
@@ -24,12 +25,13 @@
 ------------------
 
 The main way to use CFFI is as an interface to some already-compiled
-library which is provided by other means.  Imagine that you have a
-system-installed library called ``piapprox.dll`` (Windows) or
+shared object which is provided by other means.  Imagine that you have a
+system-installed shared object called ``piapprox.dll`` (Windows) or
 ``libpiapprox.so`` (Linux and others) or ``libpiapprox.dylib`` (OS X),
-containing a function ``float pi_approx(int n);`` that computes some
-approximation of pi given a number of iterations.  You want to call
-this function from Python.
+exporting a function ``float pi_approx(int n);`` that computes some
+approximation of pi given a number of iterations. You want to call
+this function from Python. Note this method works equally well with a
+static library ``piapprox.lib`` (Windows) or ``libpiapprox.a``.
 
 Create the file ``piapprox_build.py``:
 
@@ -38,13 +40,16 @@
       from cffi import FFI
       ffibuilder = FFI()
 
-      # cdef() expects a string listing the C types, functions and
-      # globals needed from Python. The string follows the C syntax.
+      # cdef() expects a single string declaring the C types, functions and
+      # globals needed to use the shared object. It must be in valid C syntax.
       ffibuilder.cdef("""
           float pi_approx(int n);
       """)
 
-      # This describes the extension module "_pi_cffi" to produce.
+      # set_source() gives the name of the python extension module to
+      # produce, and some C source code as a string.  This C code needs
+      # to make the declarated functions, types and globals available,
+      # so it is often just the "#include".
       ffibuilder.set_source("_pi_cffi",
       """
       	   #include "pi.h"   // the C header of the library
@@ -57,8 +62,8 @@
 Execute this script.  If everything is OK, it should produce
 ``_pi_cffi.c``, and then invoke the compiler on it.  The produced
 ``_pi_cffi.c`` contains a copy of the string given in ``set_source()``,
-in this example the ``#include "pi.h"``; then it contains some glue code
-for all the functions declared in the ``cdef()`` above.
+in this example the ``#include "pi.h"``. Afterwards, it contains glue code
+for all the functions, types and globals declared in the ``cdef()`` above.
 
 At runtime, you use the extension module like this:
 
@@ -113,6 +118,8 @@
 Simple example (ABI level, in-line)
 +++++++++++++++++++++++++++++++++++
 
+May look familiar to those who have used ctypes_.
+
 .. code-block:: python
 
     >>> from cffi import FFI
@@ -121,15 +128,15 @@
     ...     int printf(const char *format, ...);   // copy-pasted from the man page
     ... """)                                  
     >>> C = ffi.dlopen(None)                     # loads the entire C namespace
-    >>> arg = ffi.new("char[]", "world")         # equivalent to C code: char arg[] = "world";
-    >>> C.printf("hi there, %s.\n", arg)         # call printf
+    >>> arg = ffi.new("char[]", b"world")        # equivalent to C code: char arg[] = "world";
+    >>> C.printf(b"hi there, %s.\n", arg)        # call printf
     hi there, world.
     17                                           # this is the return value
     >>>
 
-Note that on Python 3 you need to pass byte strings to ``char *``
-arguments.  In the above example it would be ``b"world"`` and ``b"hi
-there, %s!\n"``.  In general it is ``somestring.encode(myencoding)``.
+Note that ``char *`` arguments expect a ``bytes`` object.  If you have a
+``str`` (or a ``unicode`` on Python 2) you need to encode it explicitly
+with ``somestring.encode(myencoding)``.
 
 *Python 3 on Windows:* ``ffi.dlopen(None)`` does not work.  This problem
 is messy and not really fixable.  The problem does not occur if you try
@@ -172,11 +179,12 @@
     f.close()
 
 This can be used as a more flexible replacement of the struct_ and
-array_ modules.  You could also call ``ffi.new("pixel_t[600][800]")``
+array_ modules, and replaces ctypes_.  You could also call ``ffi.new("pixel_t[600][800]")``
 and get a two-dimensional array.
 
 .. _struct: http://docs.python.org/library/struct.html
 .. _array: http://docs.python.org/library/array.html
+.. _ctypes: http://docs.python.org/library/ctypes.html
 
 *This example does not call any C compiler.*
 
@@ -430,6 +438,7 @@
     from cffi import FFI
 
     ffibuilder = FFI()
+    # Note that the actual source is None
     ffibuilder.set_source("_simple_example", None)
     ffibuilder.cdef("""
         int printf(const char *format, ...);
@@ -477,6 +486,10 @@
         install_requires=["cffi>=1.0.0"],
     )
 
+In summary, this mode is useful when you wish to declare many C structures but
+do not need fast interaction with a shared object. It is useful for parsing
+binary files, for instance.
+
 
 In-line, API level
 ++++++++++++++++++
@@ -484,7 +497,7 @@
 The "API level + in-line" mode combination exists but is long
 deprecated.  It used to be done with ``lib = ffi.verify("C header")``.
 The out-of-line variant with ``set_source("modname", "C header")`` is
-preferred.  It avoids a number of problems when the project grows in
+preferred and avoids a number of problems when the project grows in
 size.
 
 
@@ -589,8 +602,8 @@
 needs to go through the very general *libffi* library, which is slow
 (and not always perfectly tested on non-standard platforms).  The API
 mode instead compiles a CPython C wrapper that directly invokes the
-target function.  It is, comparatively, massively faster (and works
-better than libffi ever can).
+target function.  It can be massively faster (and works
+better than libffi ever will).
 
 The more fundamental reason to prefer the API mode is that *the C
 libraries are typically meant to be used with a C compiler.* You are not

From pypy.commits at gmail.com  Wed Aug 15 11:41:18 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 Aug 2018 08:41:18 -0700 (PDT)
Subject: [pypy-commit] cffi default: Update docs
Message-ID: <5b74499e.1c69fb81.804f4.7d68@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3143:d43a8bac942d
Date: 2018-08-15 17:41 +0200
http://bitbucket.org/cffi/cffi/changeset/d43a8bac942d/

Log:	Update docs

diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -34,10 +34,9 @@
 * CPython 2.6 or 2.7 or 3.x, or PyPy (PyPy 2.0 for the earliest
   versions of CFFI; or PyPy 2.6 for CFFI 1.0).
 
-* in some cases you need to be able to compile C extension modules;
-  refer to the appropriate docs for your OS.  This includes installing
-  CFFI from sources; or developing code based on ``ffi.set_source()`` or
-  ``ffi.verify()``; or installing such 3rd-party modules from sources.
+* in some cases you need to be able to compile C extension modules.
+  On non-Windows platforms, this usually means installing the package
+  ``python-dev``.  Refer to the appropriate docs for your OS.
 
 * on CPython, on non-Windows platforms, you also need to install
   ``libffi-dev`` in order to compile CFFI itself.

From pypy.commits at gmail.com  Thu Aug 16 02:14:18 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 15 Aug 2018 23:14:18 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue 2870
Message-ID: <5b75163a.1c69fb81.cc555.448a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r95005:090965df249b
Date: 2018-08-16 08:12 +0200
http://bitbucket.org/pypy/pypy/changeset/090965df249b/

Log:	Issue 2870

	Fix Windows os.listdir() for some cases (see CPython #32539 for the
	fix and the test that will show up at the next merge of the stdlib)

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -874,11 +874,11 @@
 
         if traits.str is unicode:
             if path and path[-1] not in (u'/', u'\\', u':'):
-                path += u'/'
+                path += u'\\'
             mask = path + u'*.*'
         else:
             if path and path[-1] not in ('/', '\\', ':'):
-                path += '/'
+                path += '\\'
             mask = path + '*.*'
 
         filedata = lltype.malloc(win32traits.WIN32_FIND_DATA, flavor='raw')

From pypy.commits at gmail.com  Thu Aug 16 19:51:59 2018
From: pypy.commits at gmail.com (mattip)
Date: Thu, 16 Aug 2018 16:51:59 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix test for python3 api
Message-ID: <5b760e1f.1c69fb81.13b8d.30ab@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r95006:0ebfd8239706
Date: 2018-08-17 02:50 +0300
http://bitbucket.org/pypy/pypy/changeset/0ebfd8239706/

Log:	fix test for python3 api

diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -415,7 +415,7 @@
         module = self.import_extension('foo', [
             ("enter", "METH_O",
             """
-                return PyInt_FromLong(Py_ReprEnter(args));
+                return PyLong_FromLong(Py_ReprEnter(args));
             """),
             ("leave", "METH_O",
             """

From pypy.commits at gmail.com  Thu Aug 16 19:58:52 2018
From: pypy.commits at gmail.com (mattip)
Date: Thu, 16 Aug 2018 16:58:52 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: error.strerr now returns utf8,
 uni_length; fitting for space.newtext
Message-ID: <5b760fbc.1c69fb81.3ff03.3913@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95007:45f1dd7255ba
Date: 2018-08-14 20:14 -0700
http://bitbucket.org/pypy/pypy/changeset/45f1dd7255ba/

Log:	error.strerr now returns utf8, uni_length; fitting for space.newtext

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -20,7 +20,8 @@
 def strerror(errno):
     """Translate an error code to a unicode message string."""
     from pypy.module._codecs.locale import str_decode_locale_surrogateescape
-    return str_decode_locale_surrogateescape(os.strerror(errno))
+    uni = str_decode_locale_surrogateescape(os.strerror(errno))
+    return uni.encode('utf8'), len(uni)
 
 class OperationError(Exception):
     """Interpreter-level exception that signals an exception that should be
@@ -655,12 +656,13 @@
                 return None
 
         try:
-            msg = strerror(errno)
+            msg, lgt = strerror(errno)
         except ValueError:
-            msg = u'error %d' % errno
+            msg = 'error %d' % errno
+            lgt = len(msg)
         w_errno = space.newint(errno)
         w_winerror = space.w_None
-        w_msg = space.newtext(msg.encode('utf8'), len(msg))
+        w_msg = space.newtext(msg, lgt)
 
     if w_filename is None:
         w_filename = space.w_None
@@ -690,9 +692,9 @@
                          eintr_retry=eintr_retry)
 
 def exception_from_errno(space, w_type, errno):
-    msg = strerror(errno)
+    msg, lgt = strerror(errno)
     w_error = space.call_function(w_type, space.newint(errno),
-                                  space.newtext(msg))
+                                  space.newtext(msg, lgt))
     return OperationError(w_type, w_error)
 
 def exception_from_saved_errno(space, w_type):
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -211,16 +211,16 @@
     Return value: always NULL."""
     # XXX Doesn't actually do anything with PyErr_CheckSignals.
     errno = rffi.cast(lltype.Signed, rposix._get_errno())
-    msg = _strerror(errno)
+    msg, lgt = _strerror(errno)
     if w_value:
         w_error = space.call_function(w_type,
                                       space.newint(errno),
-                                      space.newtext(msg),
+                                      space.newtext(msg, lgt),
                                       w_value)
     else:
         w_error = space.call_function(w_type,
                                       space.newint(errno),
-                                      space.newtext(msg))
+                                      space.newtext(msg, lgt))
     raise OperationError(w_type, w_error)
 
 @cpython_api([], rffi.INT_real, error=-1)
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -838,7 +838,8 @@
 def strerror(space, code):
     """Translate an error code to a message string."""
     try:
-        return space.newtext(_strerror(code))
+        # _strerror returns utf8, lgt
+        return space.newtext(*_strerror(code))
     except ValueError:
         raise oefmt(space.w_ValueError, "strerror() argument out of range")
 

From pypy.commits at gmail.com  Thu Aug 16 19:58:54 2018
From: pypy.commits at gmail.com (mattip)
Date: Thu, 16 Aug 2018 16:58:54 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: str_w -> text_w,
 raises -> pytest.raises (python3 testing artifact?)
Message-ID: <5b760fbe.1c69fb81.6044f.1ec5@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95008:945e1048b6b4
Date: 2018-08-17 02:55 +0300
http://bitbucket.org/pypy/pypy/changeset/945e1048b6b4/

Log:	str_w -> text_w, raises -> pytest.raises (python3 testing artifact?)

diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -1,4 +1,5 @@
 import py
+from pytest import raises
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from pypy.module.cpyext.api import Py_ssize_tP, PyObjectP, PyTypeObjectPtr
diff --git a/pypy/module/cpyext/test/test_import.py b/pypy/module/cpyext/test/test_import.py
--- a/pypy/module/cpyext/test/test_import.py
+++ b/pypy/module/cpyext/test/test_import.py
@@ -18,7 +18,7 @@
 
         with rffi.scoped_str2charp("foobar") as modname:
             w_foobar = PyImport_AddModule(space, modname)
-        assert space.str_w(space.getattr(w_foobar,
+        assert space.text_w(space.getattr(w_foobar,
                                          space.wrap('__name__'))) == 'foobar'
 
     def test_getmoduledict(self, space, api):
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -107,15 +107,15 @@
 
     def test_str(self, space, api):
         w_list = space.newlist([space.w_None, space.wrap(42)])
-        assert space.str_w(api.PyObject_Str(None)) == "<NULL>"
-        assert space.str_w(api.PyObject_Str(w_list)) == "[None, 42]"
-        assert space.str_w(api.PyObject_Str(space.wrap("a"))) == "a"
+        assert space.text_w(api.PyObject_Str(None)) == "<NULL>"
+        assert space.text_w(api.PyObject_Str(w_list)) == "[None, 42]"
+        assert space.text_w(api.PyObject_Str(space.wrap("a"))) == "a"
 
     def test_repr(self, space, api):
         w_list = space.newlist([space.w_None, space.wrap(42)])
-        assert space.str_w(api.PyObject_Repr(None)) == "<NULL>"
-        assert space.str_w(api.PyObject_Repr(w_list)) == "[None, 42]"
-        assert space.str_w(api.PyObject_Repr(space.wrap("a"))) == "'a'"
+        assert space.text_w(api.PyObject_Repr(None)) == "<NULL>"
+        assert space.text_w(api.PyObject_Repr(w_list)) == "[None, 42]"
+        assert space.text_w(api.PyObject_Repr(space.wrap("a"))) == "'a'"
 
     def test_RichCompare(self, space, api):
         def compare(w_o1, w_o2, opid):
@@ -204,7 +204,7 @@
 
     def test_format(self, space, api):
         w_int = space.wrap(42)
-        fmt = space.str_w(api.PyObject_Format(w_int, space.wrap('#b')))
+        fmt = space.text_w(api.PyObject_Format(w_int, space.wrap('#b')))
         assert fmt == '0b101010'
 
 class AppTestObject(AppTestCpythonExtensionBase):
diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py
--- a/pypy/module/cpyext/test/test_pyfile.py
+++ b/pypy/module/cpyext/test/test_pyfile.py
@@ -33,18 +33,18 @@
         rffi.free_charp(mode)
 
         w_line = api.PyFile_GetLine(w_file, 0)
-        assert space.str_w(w_line) == "line1\n"
+        assert space.text_w(w_line) == "line1\n"
 
         w_line = api.PyFile_GetLine(w_file, 4)
-        assert space.str_w(w_line) == "line"
+        assert space.text_w(w_line) == "line"
 
         w_line = api.PyFile_GetLine(w_file, 0)
-        assert space.str_w(w_line) == "2\n"
+        assert space.text_w(w_line) == "2\n"
 
         # XXX We ought to raise an EOFError here, but don't
         w_line = api.PyFile_GetLine(w_file, -1)
         # assert api.PyErr_Occurred() is space.w_EOFError
-        assert space.str_w(w_line) == "line3\n"
+        assert space.text_w(w_line) == "line3\n"
 
         space.call_method(w_file, "close")
 
@@ -53,7 +53,7 @@
         with rffi.scoped_str2charp(name) as filename:
             with rffi.scoped_str2charp("wb") as mode:
                 w_file = api.PyFile_FromString(filename, mode)
-        assert space.str_w(api.PyFile_Name(w_file)) == name
+        assert space.text_w(api.PyFile_Name(w_file)) == name
 
     @pytest.mark.xfail
     def test_file_setbufsize(self, space, api):
diff --git a/pypy/module/cpyext/test/test_sequence.py b/pypy/module/cpyext/test/test_sequence.py
--- a/pypy/module/cpyext/test/test_sequence.py
+++ b/pypy/module/cpyext/test/test_sequence.py
@@ -68,7 +68,7 @@
         with pytest.raises(OperationError) as excinfo:
             PySequence_Fast(space, space.wrap(3), message)
         assert excinfo.value.match(space, space.w_TypeError)
-        assert space.str_w(excinfo.value.get_w_value(space)) == "message"
+        assert space.text_w(excinfo.value.get_w_value(space)) == "message"
         rffi.free_charp(message)
 
     def test_get_slice(self, space, api):
@@ -97,7 +97,7 @@
         w_iter = api.PySeqIter_New(w_t)
         assert space.unwrap(space.next(w_iter)) == 1
         assert space.unwrap(space.next(w_iter)) == 2
-        exc = raises(OperationError, space.next, w_iter)
+        exc = pytest.raises(OperationError, space.next, w_iter)
         assert exc.value.match(space, space.w_StopIteration)
 
     def test_contains(self, space):
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -534,7 +534,7 @@
             w_s = PyUnicode_EncodeFSDefault(space, w_u)
         except OperationError:
             py.test.skip("Requires a unicode-aware fsencoding")
-        with rffi.scoped_str2charp(space.str_w(w_s)) as encoded:
+        with rffi.scoped_str2charp(space.text_w(w_s)) as encoded:
             w_decoded = PyUnicode_DecodeFSDefaultAndSize(space, encoded, space.len_w(w_s))
             assert space.eq_w(w_decoded, w_u)
             w_decoded = PyUnicode_DecodeFSDefault(space, encoded)
@@ -681,7 +681,7 @@
     def test_decode_null_encoding(self, space):
         null_charp = lltype.nullptr(rffi.CCHARP.TO)
         u_text = u'abcdefg'
-        s_text = space.str_w(PyUnicode_AsEncodedString(space, space.wrap(u_text), null_charp, null_charp))
+        s_text = space.text_w(PyUnicode_AsEncodedString(space, space.wrap(u_text), null_charp, null_charp))
         b_text = rffi.str2charp(s_text)
         assert (space.utf8_w(PyUnicode_Decode(
             space, b_text, len(s_text), null_charp, null_charp)) ==
@@ -701,7 +701,7 @@
         w_bytes = PyUnicode_EncodeMBCS(space, wbuf, 4, None)
         rffi.free_wcharp(wbuf)
         assert space.type(w_bytes) is space.w_bytes
-        assert space.str_w(w_bytes) == "abc?"
+        assert space.text_w(w_bytes) == "abc?"
 
     def test_escape(self, space):
         def test(ustr):
diff --git a/pypy/module/cpyext/test/test_userslots.py b/pypy/module/cpyext/test/test_userslots.py
--- a/pypy/module/cpyext/test/test_userslots.py
+++ b/pypy/module/cpyext/test/test_userslots.py
@@ -21,7 +21,7 @@
         assert py_datetype.c_tp_as_number.c_nb_add
         w_obj = generic_cpy_call(space, py_datetype.c_tp_as_number.c_nb_add,
                                  py_date, py_date)
-        assert space.str_w(w_obj) == 'sum!'
+        assert space.text_w(w_obj) == 'sum!'
 
     def test_tp_new_from_python(self, space, api):
         w_date = space.appexec([], """():

From pypy.commits at gmail.com  Thu Aug 16 19:58:56 2018
From: pypy.commits at gmail.com (mattip)
Date: Thu, 16 Aug 2018 16:58:56 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: fill out missing code paths
 hit by cpyext testing
Message-ID: <5b760fc0.1c69fb81.9851e.2ac2@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95009:30294b8f3847
Date: 2018-08-17 02:57 +0300
http://bitbucket.org/pypy/pypy/changeset/30294b8f3847/

Log:	fill out missing code paths hit by cpyext testing

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -221,7 +221,6 @@
                 end += 1
             res.append_slice(s, start, end)
             i = end
-    # cannot be ASCII, cannot have surrogates, I believe
     return res.build(), len(s), len(s)
 
 def utf8_encode_utf_8(s, errors, errorhandler, allow_surrogates=False):
@@ -267,8 +266,13 @@
             msg = "ordinal not in range(256)"
             res_8, newindex = errorhandler(
                 errors, 'latin1', msg, s, startindex, index)
-            result.append(res_8)
-            pos = rutf8._pos_at_index(s, newindex)
+            for cp in rutf8.Utf8StringIterator(res_8):
+                if cp > 0xFF:
+                    errorhandler("strict", 'latin1', msg, s, startindex, index)
+                result.append(chr(cp))
+            if index != newindex:  # Should be uncommon
+                index = newindex
+                pos = rutf8._pos_at_index(s, newindex)
     return result.build()
 
 def utf8_encode_ascii(s, errors, errorhandler):
@@ -649,8 +653,12 @@
             pos += 1
             continue
 
-        digits = 4 if s[pos] == 'u' else 8
-        message = "truncated \\uXXXX escape"
+        if s[pos] == 'u':
+            digits = 4
+            message = "truncated \\uXXXX escape"
+        else:
+            digits = 8
+            message = "truncated \\UXXXXXXXX escape"
         pos += 1
         pos = hexescape(builder, s, pos, digits,
                            "rawunicodeescape", errorhandler, message, errors)
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -820,10 +820,10 @@
     else:
         errors = None
 
-    result, _,  length, byteorder = str_decode_utf_16_helper(
+    result, length, pos = str_decode_utf_16_helper(
         string, errors, final=True, errorhandler=None, byteorder=byteorder)
     if pbyteorder is not None:
-        pbyteorder[0] = rffi.cast(rffi.INT_real, byteorder)
+        pbyteorder[0] = rffi.cast(rffi.INT_real, pos > 0)
     return space.newutf8(result, length)
 
 @cpython_api([CONST_STRING, Py_ssize_t, CONST_STRING, INTP_real], PyObject)
@@ -872,10 +872,10 @@
     else:
         errors = None
 
-    result, _,  length, byteorder = unicodehelper.str_decode_utf_32_helper(
-        string, errors, final=True, errorhandler=None, byteorder=byteorder)
+    result, length, pos = unicodehelper.str_decode_utf_32_helper(
+        string, errors, final=True, errorhandler='strict', byteorder=byteorder)
     if pbyteorder is not None:
-        pbyteorder[0] = rffi.cast(rffi.INT_real, byteorder)
+        pbyteorder[0] = rffi.cast(rffi.INT_real, pos>0)
     return space.newutf8(result, length)
 
 @cpython_api([rffi.CWCHARP, Py_ssize_t, rffi.CCHARP, CONST_STRING],

From pypy.commits at gmail.com  Thu Aug 16 19:58:58 2018
From: pypy.commits at gmail.com (mattip)
Date: Thu, 16 Aug 2018 16:58:58 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: merge py3.5 into branch
Message-ID: <5b760fc2.1c69fb81.2eaaa.8097@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95010:abfed62d2e22
Date: 2018-08-17 02:58 +0300
http://bitbucket.org/pypy/pypy/changeset/abfed62d2e22/

Log:	merge py3.5 into branch

diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -415,7 +415,7 @@
         module = self.import_extension('foo', [
             ("enter", "METH_O",
             """
-                return PyInt_FromLong(Py_ReprEnter(args));
+                return PyLong_FromLong(Py_ReprEnter(args));
             """),
             ("leave", "METH_O",
             """

From pypy.commits at gmail.com  Fri Aug 17 09:17:40 2018
From: pypy.commits at gmail.com (mattip)
Date: Fri, 17 Aug 2018 06:17:40 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: whoops
Message-ID: <5b76caf4.1c69fb81.5649b.d20f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95011:bec02e0b6285
Date: 2018-08-17 16:16 +0300
http://bitbucket.org/pypy/pypy/changeset/bec02e0b6285/

Log:	whoops

diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -818,10 +818,10 @@
     if llerrors:
         errors = rffi.charp2str(llerrors)
     else:
-        errors = None
+        errors = 'strict'
 
     result, length, pos = str_decode_utf_16_helper(
-        string, errors, final=True, errorhandler=None, byteorder=byteorder)
+        string, errors, True, None, byteorder=byteorder)
     if pbyteorder is not None:
         pbyteorder[0] = rffi.cast(rffi.INT_real, pos > 0)
     return space.newutf8(result, length)
@@ -870,10 +870,10 @@
     if llerrors:
         errors = rffi.charp2str(llerrors)
     else:
-        errors = None
+        errors = 'strict'
 
     result, length, pos = unicodehelper.str_decode_utf_32_helper(
-        string, errors, final=True, errorhandler='strict', byteorder=byteorder)
+        string, errors, True, None, byteorder=byteorder)
     if pbyteorder is not None:
         pbyteorder[0] = rffi.cast(rffi.INT_real, pos>0)
     return space.newutf8(result, length)
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -554,8 +554,8 @@
 
     if HAS_TM_ZONE:
         # CPython calls PyUnicode_DecodeLocale here should we do the same?
-        tm_zone = decode_utf8sp(space, rffi.charp2str(t.c_tm_zone))
-        extra = [space.newtext(tm_zone),
+        tm_zone, lgt, pos = decode_utf8sp(space, rffi.charp2str(t.c_tm_zone))
+        extra = [space.newtext(tm_zone, lgt),
                  space.newint(rffi.getintfield(t, 'c_tm_gmtoff'))]
         w_time_tuple = space.newtuple(time_tuple + extra)
     else:
@@ -578,7 +578,7 @@
         lltype.free(t_ref, flavor='raw')
         if not pbuf:
             raise OperationError(space.w_ValueError,
-                                 space.newtext(_get_error_msg()))
+                                 space.newtext(*_get_error_msg()))
         return pbuf
 
     tup_w = space.fixedview(w_tup)
@@ -744,7 +744,7 @@
 
     if not p:
         raise OperationError(space.w_ValueError,
-                             space.newtext(_get_error_msg()))
+                             space.newtext(*_get_error_msg()))
     return _tm_to_tuple(space, p)
 
 def localtime(space, w_seconds=None):
@@ -762,7 +762,7 @@
 
     if not p:
         raise OperationError(space.w_OSError,
-                             space.newtext(_get_error_msg()))
+                             space.newtext(*_get_error_msg()))
     return _tm_to_tuple(space, p)
 
 def mktime(space, w_tup):
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -218,6 +218,7 @@
     def newutf8(self, x, l):
         return w_some_obj()
 
+    @specialize.argtype(1)
     def newtext(self, x, lgt=-1):
         return w_some_obj()
     newtext_or_none = newtext

From pypy.commits at gmail.com  Sat Aug 18 14:30:14 2018
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 18 Aug 2018 11:30:14 -0700 (PDT)
Subject: [pypy-commit] pypy default: Match CPython's buggy behavior when
 importing a name with a trailing dot
Message-ID: <5b7865b6.1c69fb81.5386b.05e1@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r95012:dafacc40cdc4
Date: 2018-08-18 20:29 +0200
http://bitbucket.org/pypy/pypy/changeset/dafacc40cdc4/

Log:	Match CPython's buggy behavior when importing a name with a trailing
	dot

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -394,6 +394,8 @@
 
     w_mod = None
     parts = modulename.split('.')
+    if parts[-1] == '':
+        del parts[-1]
     prefix = []
     w_path = None
 
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -298,6 +298,13 @@
         assert ambig == sys.modules.get('ambig')
         assert hasattr(ambig,'imapackage')
 
+    def test_trailing_dot(self):
+        # bug-for-bug compatibility with CPython
+        import sys
+        __import__('pkg.pkg1.')
+        assert 'pkg.pkg1' in sys.modules
+        assert 'pkg.pkg1.' not in sys.modules
+
     def test_from_a(self):
         import sys
         from a import imamodule
@@ -751,7 +758,6 @@
             exec "from pkg.withoutall import *" in d
             assert "" in d
 
-
     def test_import_star_with_bogus___all__(self):
         for case in ["not-imported-yet", "already-imported"]:
             try:

From pypy.commits at gmail.com  Tue Aug 21 11:17:27 2018
From: pypy.commits at gmail.com (mattip)
Date: Tue, 21 Aug 2018 08:17:27 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix issue #2873 in micronumpy,
 thanks Andreas
Message-ID: <5b7c2d07.1c69fb81.b3b0.9d50@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r95013:73d842283378
Date: 2018-08-21 18:16 +0300
http://bitbucket.org/pypy/pypy/changeset/73d842283378/

Log:	fix issue #2873 in micronumpy, thanks Andreas

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -388,7 +388,7 @@
                 not self.flags & NPY.ARRAY_F_CONTIGUOUS):
            raise oefmt(errtype, "ndarray is not Fortran contiguous")
         if ((flags & space.BUF_ANY_CONTIGUOUS) == space.BUF_ANY_CONTIGUOUS and
-                not (self.flags & NPY.ARRAY_F_CONTIGUOUS and
+                not (self.flags & NPY.ARRAY_F_CONTIGUOUS or
                      self.flags & NPY.ARRAY_C_CONTIGUOUS)):
            raise oefmt(errtype, "ndarray is not contiguous")
         if ((flags & space.BUF_STRIDES) != space.BUF_STRIDES and

From pypy.commits at gmail.com  Wed Aug 22 00:29:31 2018
From: pypy.commits at gmail.com (mattip)
Date: Tue, 21 Aug 2018 21:29:31 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: fix bad merge
Message-ID: <5b7ce6ab.1c69fb81.31181.3ccd@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95014:9f29892817bc
Date: 2018-08-21 20:37 +0300
http://bitbucket.org/pypy/pypy/changeset/9f29892817bc/

Log:	fix bad merge

diff --git a/pypy/module/_multibytecodec/test/test_c_codecs.py b/pypy/module/_multibytecodec/test/test_c_codecs.py
--- a/pypy/module/_multibytecodec/test/test_c_codecs.py
+++ b/pypy/module/_multibytecodec/test/test_c_codecs.py
@@ -86,13 +86,11 @@
 def test_decode_hz_ignore():
     c = getcodec("hz")
     u = decode(c, 'def~{}abc', 'ignore')
-    assert u == u'def\u5f95'
     assert u == u'def\u5fcf'.encode('utf8')
 
 def test_decode_hz_replace():
     c = getcodec("hz")
     u = decode(c, 'def~{}abc', 'replace')
-    assert u == u'def\ufffd\u5f95\ufffd'
     assert u == u'def\ufffd\u5fcf'.encode('utf8')
 
 def test_encode_hz():

From pypy.commits at gmail.com  Wed Aug 22 00:29:33 2018
From: pypy.commits at gmail.com (mattip)
Date: Tue, 21 Aug 2018 21:29:33 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: create temp files in udir not base dir
Message-ID: <5b7ce6ad.1c69fb81.2c884.5ee1@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r95015:e2d4a1e18fbf
Date: 2018-08-21 21:04 +0300
http://bitbucket.org/pypy/pypy/changeset/e2d4a1e18fbf/

Log:	create temp files in udir not base dir

diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -20,21 +20,23 @@
         cls.w_udir = cls.space.wrap(str(udir))
 
     def w__py_file(self):
-        f = open('@TEST.py', 'w')
+        fname = self.udir + '/@TEST.py'
+        f = open(fname, 'w')
         f.write('MARKER = 42\n')
         f.close()
-        return '@TEST.py'
+        return fname
 
     def w__pyc_file(self):
         import marshal, imp
         co = compile("marker=42", "x.py", "exec")
-        f = open('@TEST.pyc', 'wb')
+        fname = self.udir + '/@TEST.pyc'
+        f = open(fname, 'wb')
         f.write(imp.get_magic())
         f.write(b'\x00\x00\x00\x00')
         f.write(b'\x00\x00\x00\x00')
         marshal.dump(co, f)
         f.close()
-        return '@TEST.pyc'
+        return fname
 
     def test_find_module(self):
         import os, imp

From pypy.commits at gmail.com  Wed Aug 22 00:29:36 2018
From: pypy.commits at gmail.com (mattip)
Date: Tue, 21 Aug 2018 21:29:36 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: merge py3.5 into branch
Message-ID: <5b7ce6b0.1c69fb81.a5329.3d91@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95016:32701764f40d
Date: 2018-08-21 21:33 +0300
http://bitbucket.org/pypy/pypy/changeset/32701764f40d/

Log:	merge py3.5 into branch

diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -20,21 +20,23 @@
         cls.w_udir = cls.space.wrap(str(udir))
 
     def w__py_file(self):
-        f = open('@TEST.py', 'w')
+        fname = self.udir + '/@TEST.py'
+        f = open(fname, 'w')
         f.write('MARKER = 42\n')
         f.close()
-        return '@TEST.py'
+        return fname
 
     def w__pyc_file(self):
         import marshal, imp
         co = compile("marker=42", "x.py", "exec")
-        f = open('@TEST.pyc', 'wb')
+        fname = self.udir + '/@TEST.pyc'
+        f = open(fname, 'wb')
         f.write(imp.get_magic())
         f.write(b'\x00\x00\x00\x00')
         f.write(b'\x00\x00\x00\x00')
         marshal.dump(co, f)
         f.close()
-        return '@TEST.pyc'
+        return fname
 
     def test_find_module(self):
         import os, imp

From pypy.commits at gmail.com  Wed Aug 22 01:38:14 2018
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Aug 2018 22:38:14 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Test and fix on Windows
 (os.get_terminal_size() failures)
Message-ID: <5b7cf6c6.1c69fb81.15bd4.2cf8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r95017:f9e3e94e62d6
Date: 2018-08-22 07:37 +0200
http://bitbucket.org/pypy/pypy/changeset/f9e3e94e62d6/

Log:	Test and fix on Windows (os.get_terminal_size() failures)

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -2407,7 +2407,7 @@
 if _WIN32:
     have_functions.append("HAVE_MS_WINDOWS")
 
-def get_terminal_size(space, w_fd=None):
+def _get_terminal_size(space, w_fd=None):
     if w_fd is None:
         fd = rfile.RFile(rfile.c_stdout(), close2=(None, None)).fileno()
     else:
@@ -2447,7 +2447,13 @@
 
             w_columns = space.newint(r_uint(winsize.c_ws_col))
             w_lines = space.newint(r_uint(winsize.c_ws_row))
+    return w_columns, w_lines
 
+def get_terminal_size(space, w_fd=None):
+    try:
+        w_columns, w_lines = _get_terminal_size(space, w_fd)
+    except OSError as e:
+        raise wrap_oserror(space, e, eintr_retry=False)
     w_tuple = space.newtuple([w_columns, w_lines])
     w_terminal_size = space.getattr(space.getbuiltinmodule(os.name),
                                     space.newtext('terminal_size'))
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -1488,6 +1488,16 @@
             raises(OSError, os.getxattr, self.path, 'user.test')
             assert os.listxattr(self.path, follow_symlinks=False) == init_names
 
+    def test_get_terminal_size(self):
+        os = self.posix
+        for args in [(), (1,), (0,), (42421,)]:
+            try:
+                w, h = os.get_terminal_size(*args)
+            except (ValueError, OSError):
+                continue
+            assert isinstance(w, int)
+            assert isinstance(h, int)
+
 
 class AppTestEnvironment(object):
     def setup_class(cls):

From pypy.commits at gmail.com  Wed Aug 22 02:09:55 2018
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Aug 2018 23:09:55 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Add a failing test. I can't figure out
 for now why the array.array test
Message-ID: <5b7cfe33.1c69fb81.1a5ac.4bde@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r95018:31265dde8699
Date: 2018-08-22 08:09 +0200
http://bitbucket.org/pypy/pypy/changeset/31265dde8699/

Log:	Add a failing test. I can't figure out for now why the array.array
	test passes but not the one with the _cffi_backend.buffer().

diff --git a/pypy/module/cpyext/test/test_buffer.py b/pypy/module/cpyext/test/test_buffer.py
--- a/pypy/module/cpyext/test/test_buffer.py
+++ b/pypy/module/cpyext/test/test_buffer.py
@@ -9,13 +9,21 @@
              void* buf;
              Py_ssize_t buf_len;
              if (PyObject_AsWriteBuffer(args, &buf, &buf_len) < 0) {
-                PyErr_SetString(PyExc_ValueError, "bad value");
+                //PyErr_SetString(PyExc_ValueError, "bad value");
                 return NULL;
              }
              return PyLong_FromLong(buf_len);
              """)])
         assert module.write_buffer_len(bytearray(b'123')) == 3
         assert module.write_buffer_len(array.array('i', [1, 2, 3])) == 12
+        #
+        import _cffi_backend
+        BChar = _cffi_backend.new_primitive_type("char")
+        BCharPtr = _cffi_backend.new_pointer_type(BChar)
+        BCharArray = _cffi_backend.new_array_type(BCharPtr, None)
+        p = _cffi_backend.newp(BCharArray, b"abcde")
+        bb = _cffi_backend.buffer(p)
+        assert module.write_buffer_len(bb) == 6
 
 
 class AppTestMmap(AppTestCpythonExtensionBase):
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -138,6 +138,7 @@
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', 'struct', 'array',
                                    'itertools', 'time', 'binascii', 'mmap',
+                                   '_cffi_backend',
                                    ])
 
     @classmethod

From pypy.commits at gmail.com  Wed Aug 22 02:48:44 2018
From: pypy.commits at gmail.com (arigo)
Date: Tue, 21 Aug 2018 23:48:44 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Issue #2874
Message-ID: <5b7d074c.1c69fb81.20819.4645@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r95019:0ba874a49cee
Date: 2018-08-22 08:46 +0200
http://bitbucket.org/pypy/pypy/changeset/0ba874a49cee/

Log:	Issue #2874

	Add a runtime check that we don't overwrite 'def buffer_w()' without
	also setting the '__buffer' flag in the TypeDef. Fix various places
	where it was missing.

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1510,10 +1510,19 @@
         if readonly and flags & self.BUF_WRITABLE == self.BUF_WRITABLE:
             raise oefmt(self.w_BufferError, "Object is not writable.")
 
+    def _try_buffer_w(self, w_obj, flags):
+        if not we_are_translated():
+            if w_obj.buffer_w.im_func != W_Root.buffer_w.im_func:
+                # when 'buffer_w()' is overridden in the subclass of
+                # W_Root, we need to specify __buffer="read" or
+                # __buffer="read-write" in the TypeDef.
+                assert type(w_obj).typedef.buffer is not None
+        return w_obj.buffer_w(self, flags)
+
     def buffer_w(self, w_obj, flags):
         # New buffer interface, returns a buffer based on flags (PyObject_GetBuffer)
         try:
-            return w_obj.buffer_w(self, flags)
+            return self._try_buffer_w(w_obj, flags)
         except BufferInterfaceNotFound:
             raise oefmt(self.w_TypeError,
                         "'%T' does not support the buffer interface", w_obj)
@@ -1521,14 +1530,14 @@
     def readbuf_w(self, w_obj):
         # Old buffer interface, returns a readonly buffer (PyObject_AsReadBuffer)
         try:
-            return w_obj.buffer_w(self, self.BUF_SIMPLE).as_readbuf()
+            return self._try_buffer_w(w_obj, self.BUF_SIMPLE).as_readbuf()
         except BufferInterfaceNotFound:
             self._getarg_error("bytes-like object", w_obj)
 
     def writebuf_w(self, w_obj):
         # Old buffer interface, returns a writeable buffer (PyObject_AsWriteBuffer)
         try:
-            return w_obj.buffer_w(self, self.BUF_WRITABLE).as_writebuf()
+            return self._try_buffer_w(w_obj, self.BUF_WRITABLE).as_writebuf()
         except (BufferInterfaceNotFound, OperationError):
             self._getarg_error("read-write bytes-like object", w_obj)
 
@@ -1562,7 +1571,7 @@
                 # NB. CPython forbids surrogates here
                 return StringBuffer(w_obj.text_w(self))
             try:
-                return w_obj.buffer_w(self, self.BUF_SIMPLE).as_readbuf()
+                return self._try_buffer_w(w_obj, self.BUF_SIMPLE).as_readbuf()
             except BufferInterfaceNotFound:
                 self._getarg_error("bytes or buffer", w_obj)
         elif code == 's#':
@@ -1574,7 +1583,7 @@
             if self.isinstance_w(w_obj, self.w_unicode):  # NB. CPython forbids
                 return w_obj.text_w(self)                 # surrogates here
             try:
-                return w_obj.buffer_w(self, self.BUF_SIMPLE).as_str()
+                return self._try_buffer_w(w_obj, self.BUF_SIMPLE).as_str()
             except BufferInterfaceNotFound:
                 self._getarg_error("bytes or read-only buffer", w_obj)
         elif code == 'w*':
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -28,6 +28,9 @@
         self.bases = bases
         # Used in cpyext to fill tp_as_buffer slots
         assert __buffer in {None, 'read-write', 'read'}, "Unknown value for __buffer"
+        for base in bases:
+            if __buffer is None:
+                __buffer = base.buffer
         self.buffer = __buffer
         self.heaptype = False
         self.hasdict = '__dict__' in rawdict
diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -156,7 +156,7 @@
     return MiniBuffer(LLBuffer(ptr, size), w_cdata)
 
 MiniBuffer.typedef = TypeDef(
-    "_cffi_backend.buffer",
+    "_cffi_backend.buffer", None, None, "read-write",
     __new__ = interp2app(MiniBuffer___new__),
     __len__ = interp2app(MiniBuffer.descr_len),
     __getitem__ = interp2app(MiniBuffer.descr_getitem),
diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py
--- a/pypy/module/_rawffi/array.py
+++ b/pypy/module/_rawffi/array.py
@@ -191,7 +191,7 @@
 
 
 W_ArrayInstance.typedef = TypeDef(
-    'ArrayInstance',
+    'ArrayInstance', None, None, "read-write",
     __repr__    = interp2app(W_ArrayInstance.descr_repr),
     __setitem__ = interp2app(W_ArrayInstance.descr_setitem),
     __getitem__ = interp2app(W_ArrayInstance.descr_getitem),
@@ -215,7 +215,7 @@
             self._free()
 
 W_ArrayInstanceAutoFree.typedef = TypeDef(
-    'ArrayInstanceAutoFree',
+    'ArrayInstanceAutoFree', None, None, "read-write",
     __repr__    = interp2app(W_ArrayInstance.descr_repr),
     __setitem__ = interp2app(W_ArrayInstance.descr_setitem),
     __getitem__ = interp2app(W_ArrayInstance.descr_getitem),
diff --git a/pypy/module/_rawffi/callback.py b/pypy/module/_rawffi/callback.py
--- a/pypy/module/_rawffi/callback.py
+++ b/pypy/module/_rawffi/callback.py
@@ -135,7 +135,7 @@
     return W_CallbackPtr(space, w_callable, w_args, w_result, flags)
 
 W_CallbackPtr.typedef = TypeDef(
-    'CallbackPtr',
+    'CallbackPtr', None, None, "read",
     __new__ = interp2app(descr_new_callbackptr),
     byptr   = interp2app(W_CallbackPtr.byptr),
     buffer  = GetSetProperty(W_CallbackPtr.getbuffer),
diff --git a/pypy/module/_rawffi/structure.py b/pypy/module/_rawffi/structure.py
--- a/pypy/module/_rawffi/structure.py
+++ b/pypy/module/_rawffi/structure.py
@@ -379,7 +379,7 @@
 
 
 W_StructureInstance.typedef = TypeDef(
-    'StructureInstance',
+    'StructureInstance', None, None, "read-write",
     __repr__    = interp2app(W_StructureInstance.descr_repr),
     __getattr__ = interp2app(W_StructureInstance.getattr),
     __setattr__ = interp2app(W_StructureInstance.setattr),
@@ -401,7 +401,7 @@
             self._free()
 
 W_StructureInstanceAutoFree.typedef = TypeDef(
-    'StructureInstanceAutoFree',
+    'StructureInstanceAutoFree', None, None, "read-write",
     __repr__    = interp2app(W_StructureInstance.descr_repr),
     __getattr__ = interp2app(W_StructureInstance.getattr),
     __setattr__ = interp2app(W_StructureInstance.setattr),
diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -654,7 +654,7 @@
     def descr__getattr__(self, space, w_key):
         return space.getattr(self.w_obj, w_key)
 
-W_GenericBox.typedef = TypeDef("numpy.generic",
+W_GenericBox.typedef = TypeDef("numpy.generic", None, None, "read-write",
     __new__ = interp2app(W_GenericBox.descr__new__.im_func),
 
     __getitem__ = interp2app(W_GenericBox.descr_getitem),
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -368,6 +368,9 @@
     def warn(self, w_msg, w_warningcls, stacklevel=2):
         pass
 
+    def _try_buffer_w(self, w_obj, flags):
+        return w_obj.buffer_w(self, flags)
+
     # ----------
 
     def translates(self, func=None, argtypes=None, seeobj_w=[], **kwds):
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -376,7 +376,7 @@
         return MockBuffer(space, self.w_list, self.w_dim, self.w_fmt, \
                           self.w_size, self.w_strides, self.w_shape)
 
-W_MockArray.typedef = TypeDef("MockArray",
+W_MockArray.typedef = TypeDef("MockArray", None, None, "read-write",
     __new__ = interp2app(W_MockArray.descr_new),
 )
 

From pypy.commits at gmail.com  Wed Aug 22 05:24:19 2018
From: pypy.commits at gmail.com (mattip)
Date: Wed, 22 Aug 2018 02:24:19 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: split complicated logic
Message-ID: <5b7d2bc3.1c69fb81.ead38.1322@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95020:740e7000561f
Date: 2018-08-22 12:22 +0300
http://bitbucket.org/pypy/pypy/changeset/740e7000561f/

Log:	split complicated logic

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -62,13 +62,7 @@
                 space.newtext(reason))
             w_res = space.call_function(w_errorhandler, w_exc)
             if (not space.isinstance_w(w_res, space.w_tuple)
-                or space.len_w(w_res) != 2
-                or not (space.isinstance_w(
-                            space.getitem(w_res, space.newint(0)),
-                            space.w_unicode) or
-                        (not decode and space.isinstance_w(
-                            space.getitem(w_res, space.newint(0)),
-                            space.w_bytes)))):
+                or space.len_w(w_res) != 2):
                 if decode:
                     msg = ("decoding error handler must return "
                            "(str, int) tuple")
@@ -78,6 +72,15 @@
                 raise OperationError(space.w_TypeError, space.newtext(msg))
 
             w_replace, w_newpos = space.fixedview(w_res, 2)
+            if not (space.isinstance_w(w_replace, space.w_unicode) or
+                (not decode and space.isinstance_w(w_replace, space.w_bytes))):
+                if decode:
+                    msg = ("decoding error handler must return "
+                           "(str, int) tuple")
+                else:
+                    msg = ("encoding error handler must return "
+                           "(str/bytes, int) tuple")
+                raise OperationError(space.w_TypeError, space.newtext(msg))
             try:
                 newpos = space.int_w(w_newpos)
             except OperationError as e:

From pypy.commits at gmail.com  Wed Aug 22 05:24:21 2018
From: pypy.commits at gmail.com (mattip)
Date: Wed, 22 Aug 2018 02:24:21 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: fix python2 version to python3
 version
Message-ID: <5b7d2bc5.1c69fb81.d0484.87e9@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95021:5a26339f55a1
Date: 2018-08-22 12:22 +0300
http://bitbucket.org/pypy/pypy/changeset/5a26339f55a1/

Log:	fix python2 version to python3 version

diff --git a/pypy/module/_multibytecodec/c_codecs.py b/pypy/module/_multibytecodec/c_codecs.py
--- a/pypy/module/_multibytecodec/c_codecs.py
+++ b/pypy/module/_multibytecodec/c_codecs.py
@@ -265,11 +265,8 @@
             replace = "?"
     else:
         assert errorcb
-        rets, end = errorcb(errors, namecb, reason,
+        replace, end = errorcb(errors, namecb, reason,
                             unicodedata, start, end)
-        codec = pypy_cjk_enc_getcodec(encodebuf)
-        lgt = rutf8.get_utf8_length(rets)
-        replace = encode(codec, rets, lgt, "strict", errorcb, namecb)
     with rffi.scoped_nonmovingbuffer(replace) as inbuf:
         r = pypy_cjk_enc_replace_on_error(encodebuf, inbuf, len(replace), end)
     if r == MBERR_NOMEMORY:

From pypy.commits at gmail.com  Wed Aug 22 05:24:24 2018
From: pypy.commits at gmail.com (mattip)
Date: Wed, 22 Aug 2018 02:24:24 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: merge py3.5 into branch
Message-ID: <5b7d2bc8.1c69fb81.9a299.6fc4@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95022:9283282e9030
Date: 2018-08-22 12:23 +0300
http://bitbucket.org/pypy/pypy/changeset/9283282e9030/

Log:	merge py3.5 into branch

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1520,10 +1520,19 @@
         if readonly and flags & self.BUF_WRITABLE == self.BUF_WRITABLE:
             raise oefmt(self.w_BufferError, "Object is not writable.")
 
+    def _try_buffer_w(self, w_obj, flags):
+        if not we_are_translated():
+            if w_obj.buffer_w.im_func != W_Root.buffer_w.im_func:
+                # when 'buffer_w()' is overridden in the subclass of
+                # W_Root, we need to specify __buffer="read" or
+                # __buffer="read-write" in the TypeDef.
+                assert type(w_obj).typedef.buffer is not None
+        return w_obj.buffer_w(self, flags)
+
     def buffer_w(self, w_obj, flags):
         # New buffer interface, returns a buffer based on flags (PyObject_GetBuffer)
         try:
-            return w_obj.buffer_w(self, flags)
+            return self._try_buffer_w(w_obj, flags)
         except BufferInterfaceNotFound:
             raise oefmt(self.w_TypeError,
                         "'%T' does not support the buffer interface", w_obj)
@@ -1531,14 +1540,14 @@
     def readbuf_w(self, w_obj):
         # Old buffer interface, returns a readonly buffer (PyObject_AsReadBuffer)
         try:
-            return w_obj.buffer_w(self, self.BUF_SIMPLE).as_readbuf()
+            return self._try_buffer_w(w_obj, self.BUF_SIMPLE).as_readbuf()
         except BufferInterfaceNotFound:
             self._getarg_error("bytes-like object", w_obj)
 
     def writebuf_w(self, w_obj):
         # Old buffer interface, returns a writeable buffer (PyObject_AsWriteBuffer)
         try:
-            return w_obj.buffer_w(self, self.BUF_WRITABLE).as_writebuf()
+            return self._try_buffer_w(w_obj, self.BUF_WRITABLE).as_writebuf()
         except (BufferInterfaceNotFound, OperationError):
             self._getarg_error("read-write bytes-like object", w_obj)
 
@@ -1572,7 +1581,7 @@
                 # NB. CPython forbids surrogates here
                 return StringBuffer(w_obj.text_w(self))
             try:
-                return w_obj.buffer_w(self, self.BUF_SIMPLE).as_readbuf()
+                return self._try_buffer_w(w_obj, self.BUF_SIMPLE).as_readbuf()
             except BufferInterfaceNotFound:
                 self._getarg_error("bytes or buffer", w_obj)
         elif code == 's#':
@@ -1584,7 +1593,7 @@
             if self.isinstance_w(w_obj, self.w_unicode):  # NB. CPython forbids
                 return w_obj.text_w(self)                 # surrogates here
             try:
-                return w_obj.buffer_w(self, self.BUF_SIMPLE).as_str()
+                return self._try_buffer_w(w_obj, self.BUF_SIMPLE).as_str()
             except BufferInterfaceNotFound:
                 self._getarg_error("bytes or read-only buffer", w_obj)
         elif code == 'w*':
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -28,6 +28,9 @@
         self.bases = bases
         # Used in cpyext to fill tp_as_buffer slots
         assert __buffer in {None, 'read-write', 'read'}, "Unknown value for __buffer"
+        for base in bases:
+            if __buffer is None:
+                __buffer = base.buffer
         self.buffer = __buffer
         self.heaptype = False
         self.hasdict = '__dict__' in rawdict
diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -156,7 +156,7 @@
     return MiniBuffer(LLBuffer(ptr, size), w_cdata)
 
 MiniBuffer.typedef = TypeDef(
-    "_cffi_backend.buffer",
+    "_cffi_backend.buffer", None, None, "read-write",
     __new__ = interp2app(MiniBuffer___new__),
     __len__ = interp2app(MiniBuffer.descr_len),
     __getitem__ = interp2app(MiniBuffer.descr_getitem),
diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py
--- a/pypy/module/_rawffi/array.py
+++ b/pypy/module/_rawffi/array.py
@@ -191,7 +191,7 @@
 
 
 W_ArrayInstance.typedef = TypeDef(
-    'ArrayInstance',
+    'ArrayInstance', None, None, "read-write",
     __repr__    = interp2app(W_ArrayInstance.descr_repr),
     __setitem__ = interp2app(W_ArrayInstance.descr_setitem),
     __getitem__ = interp2app(W_ArrayInstance.descr_getitem),
@@ -215,7 +215,7 @@
             self._free()
 
 W_ArrayInstanceAutoFree.typedef = TypeDef(
-    'ArrayInstanceAutoFree',
+    'ArrayInstanceAutoFree', None, None, "read-write",
     __repr__    = interp2app(W_ArrayInstance.descr_repr),
     __setitem__ = interp2app(W_ArrayInstance.descr_setitem),
     __getitem__ = interp2app(W_ArrayInstance.descr_getitem),
diff --git a/pypy/module/_rawffi/callback.py b/pypy/module/_rawffi/callback.py
--- a/pypy/module/_rawffi/callback.py
+++ b/pypy/module/_rawffi/callback.py
@@ -135,7 +135,7 @@
     return W_CallbackPtr(space, w_callable, w_args, w_result, flags)
 
 W_CallbackPtr.typedef = TypeDef(
-    'CallbackPtr',
+    'CallbackPtr', None, None, "read",
     __new__ = interp2app(descr_new_callbackptr),
     byptr   = interp2app(W_CallbackPtr.byptr),
     buffer  = GetSetProperty(W_CallbackPtr.getbuffer),
diff --git a/pypy/module/_rawffi/structure.py b/pypy/module/_rawffi/structure.py
--- a/pypy/module/_rawffi/structure.py
+++ b/pypy/module/_rawffi/structure.py
@@ -379,7 +379,7 @@
 
 
 W_StructureInstance.typedef = TypeDef(
-    'StructureInstance',
+    'StructureInstance', None, None, "read-write",
     __repr__    = interp2app(W_StructureInstance.descr_repr),
     __getattr__ = interp2app(W_StructureInstance.getattr),
     __setattr__ = interp2app(W_StructureInstance.setattr),
@@ -401,7 +401,7 @@
             self._free()
 
 W_StructureInstanceAutoFree.typedef = TypeDef(
-    'StructureInstanceAutoFree',
+    'StructureInstanceAutoFree', None, None, "read-write",
     __repr__    = interp2app(W_StructureInstance.descr_repr),
     __getattr__ = interp2app(W_StructureInstance.getattr),
     __setattr__ = interp2app(W_StructureInstance.setattr),
diff --git a/pypy/module/cpyext/test/test_buffer.py b/pypy/module/cpyext/test/test_buffer.py
--- a/pypy/module/cpyext/test/test_buffer.py
+++ b/pypy/module/cpyext/test/test_buffer.py
@@ -9,13 +9,21 @@
              void* buf;
              Py_ssize_t buf_len;
              if (PyObject_AsWriteBuffer(args, &buf, &buf_len) < 0) {
-                PyErr_SetString(PyExc_ValueError, "bad value");
+                //PyErr_SetString(PyExc_ValueError, "bad value");
                 return NULL;
              }
              return PyLong_FromLong(buf_len);
              """)])
         assert module.write_buffer_len(bytearray(b'123')) == 3
         assert module.write_buffer_len(array.array('i', [1, 2, 3])) == 12
+        #
+        import _cffi_backend
+        BChar = _cffi_backend.new_primitive_type("char")
+        BCharPtr = _cffi_backend.new_pointer_type(BChar)
+        BCharArray = _cffi_backend.new_array_type(BCharPtr, None)
+        p = _cffi_backend.newp(BCharArray, b"abcde")
+        bb = _cffi_backend.buffer(p)
+        assert module.write_buffer_len(bb) == 6
 
 
 class AppTestMmap(AppTestCpythonExtensionBase):
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -138,6 +138,7 @@
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', 'struct', 'array',
                                    'itertools', 'time', 'binascii', 'mmap',
+                                   '_cffi_backend',
                                    ])
 
     @classmethod
diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -656,7 +656,7 @@
     def descr__getattr__(self, space, w_key):
         return space.getattr(self.w_obj, w_key)
 
-W_GenericBox.typedef = TypeDef("numpy.generic",
+W_GenericBox.typedef = TypeDef("numpy.generic", None, None, "read-write",
     __new__ = interp2app(W_GenericBox.descr__new__.im_func),
 
     __getitem__ = interp2app(W_GenericBox.descr_getitem),
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -2408,7 +2408,7 @@
 if _WIN32:
     have_functions.append("HAVE_MS_WINDOWS")
 
-def get_terminal_size(space, w_fd=None):
+def _get_terminal_size(space, w_fd=None):
     if w_fd is None:
         fd = rfile.RFile(rfile.c_stdout(), close2=(None, None)).fileno()
     else:
@@ -2448,7 +2448,13 @@
 
             w_columns = space.newint(r_uint(winsize.c_ws_col))
             w_lines = space.newint(r_uint(winsize.c_ws_row))
+    return w_columns, w_lines
 
+def get_terminal_size(space, w_fd=None):
+    try:
+        w_columns, w_lines = _get_terminal_size(space, w_fd)
+    except OSError as e:
+        raise wrap_oserror(space, e, eintr_retry=False)
     w_tuple = space.newtuple([w_columns, w_lines])
     w_terminal_size = space.getattr(space.getbuiltinmodule(os.name),
                                     space.newtext('terminal_size'))
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -1488,6 +1488,16 @@
             raises(OSError, os.getxattr, self.path, 'user.test')
             assert os.listxattr(self.path, follow_symlinks=False) == init_names
 
+    def test_get_terminal_size(self):
+        os = self.posix
+        for args in [(), (1,), (0,), (42421,)]:
+            try:
+                w, h = os.get_terminal_size(*args)
+            except (ValueError, OSError):
+                continue
+            assert isinstance(w, int)
+            assert isinstance(h, int)
+
 
 class AppTestEnvironment(object):
     def setup_class(cls):
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -369,6 +369,9 @@
     def warn(self, w_msg, w_warningcls, stacklevel=2):
         pass
 
+    def _try_buffer_w(self, w_obj, flags):
+        return w_obj.buffer_w(self, flags)
+
     # ----------
 
     def translates(self, func=None, argtypes=None, seeobj_w=[], **kwds):
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -376,7 +376,7 @@
         return MockBuffer(space, self.w_list, self.w_dim, self.w_fmt, \
                           self.w_size, self.w_strides, self.w_shape)
 
-W_MockArray.typedef = TypeDef("MockArray",
+W_MockArray.typedef = TypeDef("MockArray", None, None, "read-write",
     __new__ = interp2app(W_MockArray.descr_new),
 )
 

From pypy.commits at gmail.com  Thu Aug 23 06:24:40 2018
From: pypy.commits at gmail.com (mattip)
Date: Thu, 23 Aug 2018 03:24:40 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: add utf8_w to TinyObjSpace
Message-ID: <5b7e8b68.1c69fb81.55e59.1f78@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95023:dbc314aacc0a
Date: 2018-08-23 07:12 +0300
http://bitbucket.org/pypy/pypy/changeset/dbc314aacc0a/

Log:	add utf8_w to TinyObjSpace

diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py
--- a/pypy/tool/pytest/objspace.py
+++ b/pypy/tool/pytest/objspace.py
@@ -86,6 +86,9 @@
     def str_w(self, w_str):
         return w_str
 
+    def utf8_w(self, w_utf8):
+        return w_utf8
+
     def bytes_w(self, w_bytes):
         return w_bytes
 

From pypy.commits at gmail.com  Thu Aug 23 06:24:42 2018
From: pypy.commits at gmail.com (mattip)
Date: Thu, 23 Aug 2018 03:24:42 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: print contents of local
 machine_config file
Message-ID: <5b7e8b6a.1c69fb81.dd5a4.f319@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95024:5061432f841e
Date: 2018-08-23 13:23 +0300
http://bitbucket.org/pypy/pypy/changeset/5061432f841e/

Log:	print contents of local machine_config file

diff --git a/testrunner/runner.py b/testrunner/runner.py
--- a/testrunner/runner.py
+++ b/testrunner/runner.py
@@ -398,6 +398,9 @@
         config_py_file = os.path.expanduser(config_py_file)
         if py.path.local(config_py_file).check(file=1):
             print >>out, "using config", config_py_file
+            if ('machine' in config_py_file:
+                with open(config_py_file, 'r') as fid:
+                    print >>out, fid.read()
             execfile(config_py_file, run_param.__dict__)
         else:
             print >>out, "ignoring non-existant config", config_py_file

From pypy.commits at gmail.com  Thu Aug 23 06:27:06 2018
From: pypy.commits at gmail.com (mattip)
Date: Thu, 23 Aug 2018 03:27:06 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: typo
Message-ID: <5b7e8bfa.1c69fb81.2c884.5fd3@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95025:f372c8123f51
Date: 2018-08-23 13:26 +0300
http://bitbucket.org/pypy/pypy/changeset/f372c8123f51/

Log:	typo

diff --git a/testrunner/runner.py b/testrunner/runner.py
--- a/testrunner/runner.py
+++ b/testrunner/runner.py
@@ -398,7 +398,7 @@
         config_py_file = os.path.expanduser(config_py_file)
         if py.path.local(config_py_file).check(file=1):
             print >>out, "using config", config_py_file
-            if ('machine' in config_py_file:
+            if 'machine' in config_py_file:
                 with open(config_py_file, 'r') as fid:
                     print >>out, fid.read()
             execfile(config_py_file, run_param.__dict__)

From pypy.commits at gmail.com  Thu Aug 23 09:35:32 2018
From: pypy.commits at gmail.com (mattip)
Date: Thu, 23 Aug 2018 06:35:32 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: backout 5061432f841e,
 add debug print around __valuedict
Message-ID: <5b7eb824.1c69fb81.6b98e.51dc@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95026:3a519921c44f
Date: 2018-08-23 16:27 +0300
http://bitbucket.org/pypy/pypy/changeset/3a519921c44f/

Log:	backout 5061432f841e, add debug print around __valuedict

diff --git a/pypy/module/thread/threadlocals.py b/pypy/module/thread/threadlocals.py
--- a/pypy/module/thread/threadlocals.py
+++ b/pypy/module/thread/threadlocals.py
@@ -96,6 +96,7 @@
             ec._signals_enabled = 1    # the main thread is enabled
             self._mainthreadident = ident
         if register_in_valuedict:
+            print 'registering thread', ident
             self._valuedict[ident] = ec
         self.raw_thread_local.set(ec)
 
@@ -110,6 +111,7 @@
                 self.raw_thread_local.set(None)
                 ident = rthread.get_ident()
                 try:
+                    print 'unregistering thread', ident
                     del self._valuedict[ident]
                 except KeyError:
                     pass
diff --git a/testrunner/runner.py b/testrunner/runner.py
--- a/testrunner/runner.py
+++ b/testrunner/runner.py
@@ -398,9 +398,6 @@
         config_py_file = os.path.expanduser(config_py_file)
         if py.path.local(config_py_file).check(file=1):
             print >>out, "using config", config_py_file
-            if 'machine' in config_py_file:
-                with open(config_py_file, 'r') as fid:
-                    print >>out, fid.read()
             execfile(config_py_file, run_param.__dict__)
         else:
             print >>out, "ignoring non-existant config", config_py_file

From pypy.commits at gmail.com  Fri Aug 24 02:36:42 2018
From: pypy.commits at gmail.com (mattip)
Date: Thu, 23 Aug 2018 23:36:42 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: try bakcing out _cffi_backend,
 preload_builtins
Message-ID: <5b7fa77a.1c69fb81.2c94b.2efc@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95028:c1e571608b66
Date: 2018-08-24 09:35 +0300
http://bitbucket.org/pypy/pypy/changeset/c1e571608b66/

Log:	try bakcing out _cffi_backend, preload_builtins

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -138,7 +138,7 @@
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', 'struct', 'array',
                                    'itertools', 'time', 'binascii', 'mmap',
-                                   '_cffi_backend',
+                                   #'_cffi_backend',
                                    ])
 
     @classmethod
@@ -222,7 +222,7 @@
         if not cls.runappdirect:
             cls.sys_info = get_cpyext_info(space)
             cls.w_debug_collect = space.wrap(interp2app(debug_collect))
-            cls.preload_builtins(space)
+            #cls.preload_builtins(space)
         else:
             def w_import_module(self, name, init=None, body='', filename=None,
                     include_dirs=None, PY_SSIZE_T_CLEAN=False):
diff --git a/pypy/module/thread/threadlocals.py b/pypy/module/thread/threadlocals.py
--- a/pypy/module/thread/threadlocals.py
+++ b/pypy/module/thread/threadlocals.py
@@ -96,7 +96,6 @@
             ec._signals_enabled = 1    # the main thread is enabled
             self._mainthreadident = ident
         if register_in_valuedict:
-            print 'registering thread', ident
             self._valuedict[ident] = ec
         self.raw_thread_local.set(ec)
 
@@ -111,7 +110,6 @@
                 self.raw_thread_local.set(None)
                 ident = rthread.get_ident()
                 try:
-                    print 'unregistering thread', ident
                     del self._valuedict[ident]
                 except KeyError:
                     pass

From pypy.commits at gmail.com  Fri Aug 24 07:10:20 2018
From: pypy.commits at gmail.com (mattip)
Date: Fri, 24 Aug 2018 04:10:20 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: re-enable _cffi_backend,
 preload_builtins still disabled
Message-ID: <5b7fe79c.1c69fb81.19d8f.a940@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95029:a06c8c1ff4bf
Date: 2018-08-24 14:09 +0300
http://bitbucket.org/pypy/pypy/changeset/a06c8c1ff4bf/

Log:	re-enable _cffi_backend, preload_builtins still disabled

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -138,7 +138,7 @@
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', 'struct', 'array',
                                    'itertools', 'time', 'binascii', 'mmap',
-                                   #'_cffi_backend',
+                                   '_cffi_backend',
                                    ])
 
     @classmethod

From pypy.commits at gmail.com  Fri Aug 24 12:16:07 2018
From: pypy.commits at gmail.com (ofekmeister)
Date: Fri, 24 Aug 2018 09:16:07 -0700 (PDT)
Subject: [pypy-commit] cffi
 ofekmeister/windows-support-py_limited_api-1534700268692: [windows] Support
 Py_LIMITED_API
Message-ID: <5b802f47.1c69fb81.eb04c.e130@mx.google.com>

Author: Ofek Lev <ofekmeister at gmail.com>
Branch: ofekmeister/windows-support-py_limited_api-1534700268692
Changeset: r3145:6cc1e30004f2
Date: 2018-08-19 17:37 +0000
http://bitbucket.org/cffi/cffi/changeset/6cc1e30004f2/

Log:	[windows] Support Py_LIMITED_API

diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h
--- a/cffi/_cffi_include.h
+++ b/cffi/_cffi_include.h
@@ -7,16 +7,6 @@
    we can learn about Py_DEBUG from pyconfig.h, but it is unclear if
    the same works for the other two macros.  Py_DEBUG implies them,
    but not the other way around.
-
-   Issue #350 is still open: on Windows, the code here causes it to link
-   with PYTHON36.DLL (for example) instead of PYTHON3.DLL.  A fix was
-   attempted in 164e526a5515 and 14ce6985e1c3, but reverted: virtualenv
-   does not make PYTHON3.DLL available, and so the "correctly" compiled
-   version would not run inside a virtualenv.  We will re-apply the fix
-   after virtualenv has been fixed for some time.  For explanation, see
-   issue #355.  For a workaround if you want PYTHON3.DLL and don't worry
-   about virtualenv, see issue #350.  See also 'py_limited_api' in
-   setuptools_ext.py.
 */
 #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API)
 #  include <pyconfig.h>

From pypy.commits at gmail.com  Fri Aug 24 12:16:09 2018
From: pypy.commits at gmail.com (ofekmeister)
Date: Fri, 24 Aug 2018 09:16:09 -0700 (PDT)
Subject: [pypy-commit] cffi
 ofekmeister/windows-support-py_limited_api-1534700268692: setuptools_ext.py
 edited online with Bitbucket
Message-ID: <5b802f49.1c69fb81.cd97f.14fc@mx.google.com>

Author: Ofek Lev <ofekmeister at gmail.com>
Branch: ofekmeister/windows-support-py_limited_api-1534700268692
Changeset: r3146:21ed8f05b4a3
Date: 2018-08-19 17:42 +0000
http://bitbucket.org/cffi/cffi/changeset/21ed8f05b4a3/

Log:	setuptools_ext.py edited online with Bitbucket

diff --git a/cffi/setuptools_ext.py b/cffi/setuptools_ext.py
--- a/cffi/setuptools_ext.py
+++ b/cffi/setuptools_ext.py
@@ -81,13 +81,8 @@
     it doesn't so far, creating troubles.  That's why we check
     for "not hasattr(sys, 'gettotalrefcount')" (the 2.7 compatible equivalent
     of 'd' not in sys.abiflags). (http://bugs.python.org/issue28401)
-
-    On Windows, it's better not to use py_limited_api until issue #355
-    can be resolved (by having virtualenv copy PYTHON3.DLL).  See also
-    the start of _cffi_include.h.
     """
-    if ('py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount')
-            and sys.platform != 'win32'):
+    if 'py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount'):
         import setuptools
         try:
             setuptools_major_version = int(setuptools.__version__.partition('.')[0])

From pypy.commits at gmail.com  Fri Aug 24 12:16:11 2018
From: pypy.commits at gmail.com (arigo)
Date: Fri, 24 Aug 2018 09:16:11 -0700 (PDT)
Subject: [pypy-commit] cffi
 ofekmeister/windows-support-py_limited_api-1534700268692: Redo 164e526a5515
 and 14ce6985e1c3.
Message-ID: <5b802f4b.1c69fb81.32137.de60@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: ofekmeister/windows-support-py_limited_api-1534700268692
Changeset: r3147:996c463c0b1d
Date: 2018-08-24 18:11 +0200
http://bitbucket.org/cffi/cffi/changeset/996c463c0b1d/

Log:	Redo 164e526a5515 and 14ce6985e1c3.

diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h
--- a/cffi/_cffi_include.h
+++ b/cffi/_cffi_include.h
@@ -7,11 +7,44 @@
    we can learn about Py_DEBUG from pyconfig.h, but it is unclear if
    the same works for the other two macros.  Py_DEBUG implies them,
    but not the other way around.
+
+   The implementation is messy (issue #350): on Windows, with _MSC_VER,
+   we have to define Py_LIMITED_API even before including pyconfig.h.
+   In that case, we guess what pyconfig.h will do to the macros above,
+   and check our guess after the #include.
+
+   Note that on Windows, with CPython 3.x, you need virtualenv version
+   >= 16.0.0.  Older versions don't copy PYTHON3.DLL.  As a workaround
+   you can remove the definition of Py_LIMITED_API here.
+
+   See also 'py_limited_api' in cffi/setuptools_ext.py.
 */
 #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API)
-#  include <pyconfig.h>
-#  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
-#    define Py_LIMITED_API
+#  ifdef _MSC_VER
+#    if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+#      define Py_LIMITED_API
+#    endif
+#    include <pyconfig.h>
+     /* sanity-check: Py_LIMITED_API will cause crashes if any of these
+        are also defined.  Normally, the Python file PC/pyconfig.h does not
+        cause any of these to be defined, with the exception that _DEBUG
+        causes Py_DEBUG.  Double-check that. */
+#    ifdef Py_LIMITED_API
+#      if defined(Py_DEBUG)
+#        error "pyconfig.h unexpectedly defines Py_DEBUG, but Py_LIMITED_API is set"
+#      endif
+#      if defined(Py_TRACE_REFS)
+#        error "pyconfig.h unexpectedly defines Py_TRACE_REFS, but Py_LIMITED_API is set"
+#      endif
+#      if defined(Py_REF_DEBUG)
+#        error "pyconfig.h unexpectedly defines Py_REF_DEBUG, but Py_LIMITED_API is set"
+#      endif
+#    endif
+#  else
+#    include <pyconfig.h>
+#    if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+#      define Py_LIMITED_API
+#    endif
 #  endif
 #endif
 

From pypy.commits at gmail.com  Fri Aug 24 12:16:13 2018
From: pypy.commits at gmail.com (arigo)
Date: Fri, 24 Aug 2018 09:16:13 -0700 (PDT)
Subject: [pypy-commit] cffi
 ofekmeister/windows-support-py_limited_api-1534700268692: Document it in
 whatsnew
Message-ID: <5b802f4d.1c69fb81.2f3c.8357@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: ofekmeister/windows-support-py_limited_api-1534700268692
Changeset: r3148:9482bcee722e
Date: 2018-08-24 18:15 +0200
http://bitbucket.org/cffi/cffi/changeset/9482bcee722e/

Log:	Document it in whatsnew

diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -14,6 +14,10 @@
   to ``pack=1`` (saying e.g. that fields like ``int`` should be aligned
   to 1 byte instead of 4).
 
+* Windows, CPython 3.x: link cffi modules with ``python3.dll``
+  again.  This makes them independant on the exact CPython version,
+  like they are on other platforms.  **It requires virtualenv 16.0.0.**
+
 
 v1.11.5
 =======

From pypy.commits at gmail.com  Sun Aug 26 16:00:21 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 26 Aug 2018 13:00:21 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: try making _cffi_backend first
Message-ID: <5b8306d5.1c69fb81.160ed.6bc9@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95030:caa93d668ded
Date: 2018-08-26 21:59 +0200
http://bitbucket.org/pypy/pypy/changeset/caa93d668ded/

Log:	try making _cffi_backend first

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -136,9 +136,9 @@
 
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
-    spaceconfig = dict(usemodules=['cpyext', 'thread', 'struct', 'array',
-                                   'itertools', 'time', 'binascii', 'mmap',
-                                   '_cffi_backend',
+    spaceconfig = dict(usemodules=['_cffi_backend', 'cpyext', 'thread',
+                                   'struct', 'array', 'itertools', 'time',
+                                   'binascii', 'mmap',
                                    ])
 
     @classmethod

From pypy.commits at gmail.com  Mon Aug 27 16:24:34 2018
From: pypy.commits at gmail.com (arigo)
Date: Mon, 27 Aug 2018 13:24:34 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix for cpyext test_translate when run
 after test_buffer. Not sure why
Message-ID: <5b845e02.1c69fb81.dec3e.1631@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r95032:06709625320e
Date: 2018-08-27 22:23 +0200
http://bitbucket.org/pypy/pypy/changeset/06709625320e/

Log:	Fix for cpyext test_translate when run after test_buffer. Not sure
	why but it must be something like the thread support in the
	embedding mode of cffi.

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -200,6 +200,10 @@
                default=False,
                requires=[("objspace.usemodules.cpyext", False)]),
 
+    BoolOption("disable_entrypoints_in_cffi",
+               "Disable only cffi's embedding mode.",
+               default=False),
+
     BoolOption("fstrings",
                "if you are really convinced that f-strings are a security "
                "issue, you can disable them here",
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -71,7 +71,8 @@
     def __init__(self, space, *args):
         MixedModule.__init__(self, space, *args)
         #
-        if not space.config.objspace.disable_entrypoints:
+        if (not space.config.objspace.disable_entrypoints and
+            not space.config.objspace.disable_entrypoints_in_cffi):
             # import 'embedding', which has the side-effect of registering
             # the 'pypy_init_embedded_cffi_module' entry point
             from pypy.module._cffi_backend import embedding
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -136,10 +136,11 @@
 
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
-    spaceconfig = dict(usemodules=['cpyext', 'thread', 'struct', 'array',
+    spaceconfig = {"usemodules" : ['cpyext', 'thread', 'struct', 'array',
                                    'itertools', 'time', 'binascii', 'mmap',
                                    '_cffi_backend',
-                                   ])
+                                   ],
+                   "objspace.disable_entrypoints_in_cffi": True}
 
     @classmethod
     def preload_builtins(cls, space):

From pypy.commits at gmail.com  Mon Aug 27 17:04:46 2018
From: pypy.commits at gmail.com (arigo)
Date: Mon, 27 Aug 2018 14:04:46 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Skip this test unless run in -A mode
 (again, not completely sure I understand everything but it's messy)
Message-ID: <5b84676e.1c69fb81.2cc76.138d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r95033:530cd9055fea
Date: 2018-08-27 23:02 +0200
http://bitbucket.org/pypy/pypy/changeset/530cd9055fea/

Log:	Skip this test unless run in -A mode (again, not completely sure I
	understand everything but it's messy)

diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -479,6 +479,7 @@
             ])
         raises(SystemError, module.oops)
 
+    @pytest.mark.skipif("not config.option.runappdirect", reason='-A only')
     def test_error_thread_race(self):
         # Check race condition: thread 0 returns from cpyext with error set,
         # after thread 1 has set an error but before it returns.

From pypy.commits at gmail.com  Wed Aug 29 03:49:26 2018
From: pypy.commits at gmail.com (mattip)
Date: Wed, 29 Aug 2018 00:49:26 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: merge py3.5 into branch
Message-ID: <5b865006.1c69fb81.c0079.1422@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95034:d39037807956
Date: 2018-08-28 06:59 +0200
http://bitbucket.org/pypy/pypy/changeset/d39037807956/

Log:	merge py3.5 into branch

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -200,6 +200,10 @@
                default=False,
                requires=[("objspace.usemodules.cpyext", False)]),
 
+    BoolOption("disable_entrypoints_in_cffi",
+               "Disable only cffi's embedding mode.",
+               default=False),
+
     BoolOption("fstrings",
                "if you are really convinced that f-strings are a security "
                "issue, you can disable them here",
diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py
--- a/pypy/interpreter/pyparser/pyparse.py
+++ b/pypy/interpreter/pyparser/pyparse.py
@@ -236,7 +236,7 @@
                             next_token_seen is not None and
                             next_token_seen.value != '('):
                         msg = "Missing parentheses in call to '%s'" % (
-                            last_token_seen,)
+                            last_token_seen.value,)
                     else:
                         msg = "invalid syntax"
                     if e.expected_str is not None:
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -71,7 +71,8 @@
     def __init__(self, space, *args):
         MixedModule.__init__(self, space, *args)
         #
-        if not space.config.objspace.disable_entrypoints:
+        if (not space.config.objspace.disable_entrypoints and
+            not space.config.objspace.disable_entrypoints_in_cffi):
             # import 'embedding', which has the side-effect of registering
             # the 'pypy_init_embedded_cffi_module' entry point
             from pypy.module._cffi_backend import embedding
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -136,10 +136,11 @@
 
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
-    spaceconfig = dict(usemodules=['_cffi_backend', 'cpyext', 'thread',
-                                   'struct', 'array', 'itertools', 'time',
-                                   'binascii', 'mmap',
-                                   ])
+    spaceconfig = {"usemodules" : ['cpyext', 'thread', 'struct', 'array',
+                                   'itertools', 'time', 'binascii', 'mmap',
+                                   '_cffi_backend',
+                                   ],
+                   "objspace.disable_entrypoints_in_cffi": True}
 
     @classmethod
     def preload_builtins(cls, space):
@@ -222,7 +223,7 @@
         if not cls.runappdirect:
             cls.sys_info = get_cpyext_info(space)
             cls.w_debug_collect = space.wrap(interp2app(debug_collect))
-            #cls.preload_builtins(space)
+            cls.preload_builtins(space)
         else:
             def w_import_module(self, name, init=None, body='', filename=None,
                     include_dirs=None, PY_SSIZE_T_CLEAN=False):
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -479,6 +479,7 @@
             ])
         raises(SystemError, module.oops)
 
+    @pytest.mark.skipif("not config.option.runappdirect", reason='-A only')
     def test_error_thread_race(self):
         # Check race condition: thread 0 returns from cpyext with error set,
         # after thread 1 has set an error but before it returns.

From pypy.commits at gmail.com  Wed Aug 29 03:49:29 2018
From: pypy.commits at gmail.com (mattip)
Date: Wed, 29 Aug 2018 00:49:29 -0700 (PDT)
Subject: [pypy-commit] pypy default: split tests into two directories to
 speed up parallel testing
Message-ID: <5b865009.1c69fb81.c476f.e66d@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r95035:943b0266d564
Date: 2018-08-29 08:16 +0200
http://bitbucket.org/pypy/pypy/changeset/943b0266d564/

Log:	split tests into two directories to speed up parallel testing

diff --git a/pypy/module/cpyext/test0/__init__.py b/pypy/module/cpyext/test0/__init__.py
new file mode 100644
diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test0/conftest.py
copy from pypy/module/cpyext/test/conftest.py
copy to pypy/module/cpyext/test0/conftest.py
diff --git a/pypy/module/cpyext/test/test_abstract.py b/pypy/module/cpyext/test0/test_abstract.py
rename from pypy/module/cpyext/test/test_abstract.py
rename to pypy/module/cpyext/test0/test_abstract.py
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test0/test_arraymodule.py
rename from pypy/module/cpyext/test/test_arraymodule.py
rename to pypy/module/cpyext/test0/test_arraymodule.py
diff --git a/pypy/module/cpyext/test/test_boolobject.py b/pypy/module/cpyext/test0/test_boolobject.py
rename from pypy/module/cpyext/test/test_boolobject.py
rename to pypy/module/cpyext/test0/test_boolobject.py
diff --git a/pypy/module/cpyext/test/test_borrow.py b/pypy/module/cpyext/test0/test_borrow.py
rename from pypy/module/cpyext/test/test_borrow.py
rename to pypy/module/cpyext/test0/test_borrow.py
diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test0/test_bufferobject.py
rename from pypy/module/cpyext/test/test_bufferobject.py
rename to pypy/module/cpyext/test0/test_bufferobject.py
diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test0/test_bytearrayobject.py
rename from pypy/module/cpyext/test/test_bytearrayobject.py
rename to pypy/module/cpyext/test0/test_bytearrayobject.py
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test0/test_bytesobject.py
rename from pypy/module/cpyext/test/test_bytesobject.py
rename to pypy/module/cpyext/test0/test_bytesobject.py
diff --git a/pypy/module/cpyext/test/test_capsule.py b/pypy/module/cpyext/test0/test_capsule.py
rename from pypy/module/cpyext/test/test_capsule.py
rename to pypy/module/cpyext/test0/test_capsule.py
diff --git a/pypy/module/cpyext/test/test_cell.py b/pypy/module/cpyext/test0/test_cell.py
rename from pypy/module/cpyext/test/test_cell.py
rename to pypy/module/cpyext/test0/test_cell.py
diff --git a/pypy/module/cpyext/test/test_classobject.py b/pypy/module/cpyext/test0/test_classobject.py
rename from pypy/module/cpyext/test/test_classobject.py
rename to pypy/module/cpyext/test0/test_classobject.py
diff --git a/pypy/module/cpyext/test/test_codecs.py b/pypy/module/cpyext/test0/test_codecs.py
rename from pypy/module/cpyext/test/test_codecs.py
rename to pypy/module/cpyext/test0/test_codecs.py
diff --git a/pypy/module/cpyext/test/test_complexobject.py b/pypy/module/cpyext/test0/test_complexobject.py
rename from pypy/module/cpyext/test/test_complexobject.py
rename to pypy/module/cpyext/test0/test_complexobject.py
diff --git a/pypy/module/cpyext/test/test_cparser.py b/pypy/module/cpyext/test0/test_cparser.py
rename from pypy/module/cpyext/test/test_cparser.py
rename to pypy/module/cpyext/test0/test_cparser.py
diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test0/test_datetime.py
rename from pypy/module/cpyext/test/test_datetime.py
rename to pypy/module/cpyext/test0/test_datetime.py
diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test0/test_dictobject.py
rename from pypy/module/cpyext/test/test_dictobject.py
rename to pypy/module/cpyext/test0/test_dictobject.py
diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test0/test_eval.py
rename from pypy/module/cpyext/test/test_eval.py
rename to pypy/module/cpyext/test0/test_eval.py
diff --git a/pypy/module/cpyext/test/test_fileobject.py b/pypy/module/cpyext/test0/test_fileobject.py
rename from pypy/module/cpyext/test/test_fileobject.py
rename to pypy/module/cpyext/test0/test_fileobject.py
diff --git a/pypy/module/cpyext/test/test_floatobject.py b/pypy/module/cpyext/test0/test_floatobject.py
rename from pypy/module/cpyext/test/test_floatobject.py
rename to pypy/module/cpyext/test0/test_floatobject.py
diff --git a/pypy/module/cpyext/test/test_frameobject.py b/pypy/module/cpyext/test0/test_frameobject.py
rename from pypy/module/cpyext/test/test_frameobject.py
rename to pypy/module/cpyext/test0/test_frameobject.py
diff --git a/pypy/module/cpyext/test/test_funcobject.py b/pypy/module/cpyext/test0/test_funcobject.py
rename from pypy/module/cpyext/test/test_funcobject.py
rename to pypy/module/cpyext/test0/test_funcobject.py
diff --git a/pypy/module/cpyext/test/test_genobject.py b/pypy/module/cpyext/test0/test_genobject.py
rename from pypy/module/cpyext/test/test_genobject.py
rename to pypy/module/cpyext/test0/test_genobject.py
diff --git a/pypy/module/cpyext/test/test_getargs.py b/pypy/module/cpyext/test0/test_getargs.py
rename from pypy/module/cpyext/test/test_getargs.py
rename to pypy/module/cpyext/test0/test_getargs.py
diff --git a/pypy/module/cpyext/test/test_import.py b/pypy/module/cpyext/test0/test_import.py
rename from pypy/module/cpyext/test/test_import.py
rename to pypy/module/cpyext/test0/test_import.py
diff --git a/pypy/module/cpyext/test/test_intobject.py b/pypy/module/cpyext/test0/test_intobject.py
rename from pypy/module/cpyext/test/test_intobject.py
rename to pypy/module/cpyext/test0/test_intobject.py
diff --git a/pypy/module/cpyext/test/test_iterator.py b/pypy/module/cpyext/test0/test_iterator.py
rename from pypy/module/cpyext/test/test_iterator.py
rename to pypy/module/cpyext/test0/test_iterator.py
diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test0/test_listobject.py
rename from pypy/module/cpyext/test/test_listobject.py
rename to pypy/module/cpyext/test0/test_listobject.py
diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test0/test_longobject.py
rename from pypy/module/cpyext/test/test_longobject.py
rename to pypy/module/cpyext/test0/test_longobject.py
diff --git a/pypy/module/cpyext/test/test_mapping.py b/pypy/module/cpyext/test0/test_mapping.py
rename from pypy/module/cpyext/test/test_mapping.py
rename to pypy/module/cpyext/test0/test_mapping.py
diff --git a/pypy/module/cpyext/test/test_marshal.py b/pypy/module/cpyext/test0/test_marshal.py
rename from pypy/module/cpyext/test/test_marshal.py
rename to pypy/module/cpyext/test0/test_marshal.py
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test0/test_memoryobject.py
rename from pypy/module/cpyext/test/test_memoryobject.py
rename to pypy/module/cpyext/test0/test_memoryobject.py
diff --git a/pypy/module/cpyext/test/test_methodobject.py b/pypy/module/cpyext/test0/test_methodobject.py
rename from pypy/module/cpyext/test/test_methodobject.py
rename to pypy/module/cpyext/test0/test_methodobject.py
diff --git a/pypy/module/cpyext/test/test_module.py b/pypy/module/cpyext/test0/test_module.py
rename from pypy/module/cpyext/test/test_module.py
rename to pypy/module/cpyext/test0/test_module.py
diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test0/test_ndarrayobject.py
rename from pypy/module/cpyext/test/test_ndarrayobject.py
rename to pypy/module/cpyext/test0/test_ndarrayobject.py
diff --git a/pypy/module/cpyext/test/test_number.py b/pypy/module/cpyext/test0/test_number.py
rename from pypy/module/cpyext/test/test_number.py
rename to pypy/module/cpyext/test0/test_number.py
diff --git a/pypy/module/cpyext/test1/__init__.py b/pypy/module/cpyext/test1/__init__.py
new file mode 100644
diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test1/conftest.py
copy from pypy/module/cpyext/test/conftest.py
copy to pypy/module/cpyext/test1/conftest.py
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test1/test_object.py
rename from pypy/module/cpyext/test/test_object.py
rename to pypy/module/cpyext/test1/test_object.py
diff --git a/pypy/module/cpyext/test/test_pycobject.py b/pypy/module/cpyext/test1/test_pycobject.py
rename from pypy/module/cpyext/test/test_pycobject.py
rename to pypy/module/cpyext/test1/test_pycobject.py
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test1/test_pyerrors.py
rename from pypy/module/cpyext/test/test_pyerrors.py
rename to pypy/module/cpyext/test1/test_pyerrors.py
diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test1/test_pyfile.py
rename from pypy/module/cpyext/test/test_pyfile.py
rename to pypy/module/cpyext/test1/test_pyfile.py
diff --git a/pypy/module/cpyext/test/test_pysignals.py b/pypy/module/cpyext/test1/test_pysignals.py
rename from pypy/module/cpyext/test/test_pysignals.py
rename to pypy/module/cpyext/test1/test_pysignals.py
diff --git a/pypy/module/cpyext/test/test_pystate.py b/pypy/module/cpyext/test1/test_pystate.py
rename from pypy/module/cpyext/test/test_pystate.py
rename to pypy/module/cpyext/test1/test_pystate.py
diff --git a/pypy/module/cpyext/test/test_pystrtod.py b/pypy/module/cpyext/test1/test_pystrtod.py
rename from pypy/module/cpyext/test/test_pystrtod.py
rename to pypy/module/cpyext/test1/test_pystrtod.py
diff --git a/pypy/module/cpyext/test/test_sequence.py b/pypy/module/cpyext/test1/test_sequence.py
rename from pypy/module/cpyext/test/test_sequence.py
rename to pypy/module/cpyext/test1/test_sequence.py
diff --git a/pypy/module/cpyext/test/test_setobject.py b/pypy/module/cpyext/test1/test_setobject.py
rename from pypy/module/cpyext/test/test_setobject.py
rename to pypy/module/cpyext/test1/test_setobject.py
diff --git a/pypy/module/cpyext/test/test_sliceobject.py b/pypy/module/cpyext/test1/test_sliceobject.py
rename from pypy/module/cpyext/test/test_sliceobject.py
rename to pypy/module/cpyext/test1/test_sliceobject.py
diff --git a/pypy/module/cpyext/test/test_structseq.py b/pypy/module/cpyext/test1/test_structseq.py
rename from pypy/module/cpyext/test/test_structseq.py
rename to pypy/module/cpyext/test1/test_structseq.py
diff --git a/pypy/module/cpyext/test/test_sysmodule.py b/pypy/module/cpyext/test1/test_sysmodule.py
rename from pypy/module/cpyext/test/test_sysmodule.py
rename to pypy/module/cpyext/test1/test_sysmodule.py
diff --git a/pypy/module/cpyext/test/test_thread.py b/pypy/module/cpyext/test1/test_thread.py
rename from pypy/module/cpyext/test/test_thread.py
rename to pypy/module/cpyext/test1/test_thread.py
diff --git a/pypy/module/cpyext/test/test_traceback.py b/pypy/module/cpyext/test1/test_traceback.py
rename from pypy/module/cpyext/test/test_traceback.py
rename to pypy/module/cpyext/test1/test_traceback.py
diff --git a/pypy/module/cpyext/test/test_translate.py b/pypy/module/cpyext/test1/test_translate.py
rename from pypy/module/cpyext/test/test_translate.py
rename to pypy/module/cpyext/test1/test_translate.py
diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test1/test_tupleobject.py
rename from pypy/module/cpyext/test/test_tupleobject.py
rename to pypy/module/cpyext/test1/test_tupleobject.py
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test1/test_typeobject.py
rename from pypy/module/cpyext/test/test_typeobject.py
rename to pypy/module/cpyext/test1/test_typeobject.py
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test1/test_unicodeobject.py
rename from pypy/module/cpyext/test/test_unicodeobject.py
rename to pypy/module/cpyext/test1/test_unicodeobject.py
diff --git a/pypy/module/cpyext/test/test_userslots.py b/pypy/module/cpyext/test1/test_userslots.py
rename from pypy/module/cpyext/test/test_userslots.py
rename to pypy/module/cpyext/test1/test_userslots.py
diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test1/test_version.py
rename from pypy/module/cpyext/test/test_version.py
rename to pypy/module/cpyext/test1/test_version.py
diff --git a/pypy/module/cpyext/test/test_weakref.py b/pypy/module/cpyext/test1/test_weakref.py
rename from pypy/module/cpyext/test/test_weakref.py
rename to pypy/module/cpyext/test1/test_weakref.py

From pypy.commits at gmail.com  Wed Aug 29 03:49:31 2018
From: pypy.commits at gmail.com (mattip)
Date: Wed, 29 Aug 2018 00:49:31 -0700 (PDT)
Subject: [pypy-commit] pypy default: ensure datetimeAPI is initialized,
 for subclassing cpyext classes
Message-ID: <5b86500b.1c69fb81.86623.5336@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r95036:d2dd59afa85c
Date: 2018-08-29 08:16 +0200
http://bitbucket.org/pypy/pypy/changeset/d2dd59afa85c/

Log:	ensure datetimeAPI is initialized, for subclassing cpyext classes

diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py
--- a/pypy/module/cpyext/cdatetime.py
+++ b/pypy/module/cpyext/cdatetime.py
@@ -137,8 +137,9 @@
     If it is a datetime.time or datetime.datetime, it may have tzinfo
     '''
     state = space.fromcache(State)
-    # cannot raise here, so just crash
-    assert len(state.datetimeAPI) > 0
+    if len(state.datetimeAPI) ==0:
+        # can happen in subclassing
+        _PyDateTime_Import(space)
     if state.datetimeAPI[0].c_TimeType == py_obj.c_ob_type:
         py_datetime = rffi.cast(PyDateTime_Time, py_obj)
         w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo'))

From pypy.commits at gmail.com  Wed Aug 29 03:49:34 2018
From: pypy.commits at gmail.com (mattip)
Date: Wed, 29 Aug 2018 00:49:34 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merge default into py3.5
Message-ID: <5b86500e.1c69fb81.846a6.ea69@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r95037:c76008be29e5
Date: 2018-08-29 08:17 +0200
http://bitbucket.org/pypy/pypy/changeset/c76008be29e5/

Log:	merge default into py3.5

diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py
--- a/pypy/module/cpyext/cdatetime.py
+++ b/pypy/module/cpyext/cdatetime.py
@@ -137,8 +137,9 @@
     If it is a datetime.time or datetime.datetime, it may have tzinfo
     '''
     state = space.fromcache(State)
-    # cannot raise here, so just crash
-    assert len(state.datetimeAPI) > 0
+    if len(state.datetimeAPI) ==0:
+        # can happen in subclassing
+        _PyDateTime_Import(space)
     if state.datetimeAPI[0].c_TimeType == py_obj.c_ob_type:
         py_datetime = rffi.cast(PyDateTime_Time, py_obj)
         w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo'))
diff --git a/pypy/module/cpyext/include/pymath.h b/pypy/module/cpyext/include/pymath.h
--- a/pypy/module/cpyext/include/pymath.h
+++ b/pypy/module/cpyext/include/pymath.h
@@ -6,6 +6,74 @@
 functions and constants
 **************************************************************************/
 
+/* High precision definition of pi and e (Euler)
+ * The values are taken from libc6's math.h.
+ */
+#ifndef Py_MATH_PIl
+#define Py_MATH_PIl 3.1415926535897932384626433832795029L
+#endif
+#ifndef Py_MATH_PI
+#define Py_MATH_PI 3.14159265358979323846
+#endif
+
+#ifndef Py_MATH_El
+#define Py_MATH_El 2.7182818284590452353602874713526625L
+#endif
+
+#ifndef Py_MATH_E
+#define Py_MATH_E 2.7182818284590452354
+#endif
+
+/* Tau (2pi) to 40 digits, taken from tauday.com/tau-digits. */
+#ifndef Py_MATH_TAU
+#define Py_MATH_TAU 6.2831853071795864769252867665590057683943L
+#endif
+
+/* Py_IS_NAN(X)
+ * Return 1 if float or double arg is a NaN, else 0.
+ * Caution:
+ *     X is evaluated more than once.
+ *     This may not work on all platforms.  Each platform has *some*
+ *     way to spell this, though -- override in pyconfig.h if you have
+ *     a platform where it doesn't work.
+ */
+#ifndef Py_IS_NAN
+#if __STDC_VERSION__ >= 199901L || __cplusplus >= 201103L
+#define Py_IS_NAN(X) isnan(X)
+#else
+#define Py_IS_NAN(X) ((X) != (X))
+#endif
+#endif
+
+/* Py_IS_INFINITY(X)
+ * Return 1 if float or double arg is an infinity, else 0.
+ * Caution:
+ *    X is evaluated more than once.
+ *    This implementation may set the underflow flag if |X| is very small;
+ *    it really can't be implemented correctly (& easily) before C99.
+ *    Override in pyconfig.h if you have a better spelling on your platform.
+ */
+#ifndef Py_IS_INFINITY
+#  if __STDC_VERSION__ >= 199901L || __cplusplus >= 201103L
+#    define Py_IS_INFINITY(X) isinf(X)
+#  else
+#    define Py_IS_INFINITY(X) ((X) && ((X)*0.5 == (X)))
+#  endif
+#endif
+
+/* Py_IS_FINITE(X)
+ * Return 1 if float or double arg is neither infinite nor NAN, else 0.
+ * Some compilers (e.g. VisualStudio) have intrisics for this, so a special
+ * macro for this particular test is useful
+ */
+#ifndef Py_IS_FINITE
+#if __STDC_VERSION__ >= 199901L || __cplusplus >= 201103L
+#define Py_IS_FINITE(X) isfinite(X)
+#else
+#define Py_IS_FINITE(X) (!Py_IS_INFINITY(X) && !Py_IS_NAN(X))
+#endif
+#endif
+
 /* HUGE_VAL is supposed to expand to a positive double infinity.  Python
  * uses Py_HUGE_VAL instead because some platforms are broken in this
  * respect.  We used to embed code in pyport.h to try to worm around that,
diff --git a/pypy/module/cpyext/test/test_floatobject.py b/pypy/module/cpyext/test/test_floatobject.py
deleted file mode 100644
--- a/pypy/module/cpyext/test/test_floatobject.py
+++ /dev/null
@@ -1,112 +0,0 @@
-import pytest
-from pypy.interpreter.error import OperationError
-from pypy.module.cpyext.test.test_api import BaseApiTest
-from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from rpython.rtyper.lltypesystem import rffi
-from pypy.module.cpyext.floatobject import (
-    PyFloat_FromDouble, PyFloat_AsDouble, PyFloat_AS_DOUBLE, PyNumber_Float,
-    _PyFloat_Unpack4, _PyFloat_Unpack8)
-
-class TestFloatObject(BaseApiTest):
-    def test_floatobject(self, space):
-        assert space.unwrap(PyFloat_FromDouble(space, 3.14)) == 3.14
-        assert PyFloat_AsDouble(space, space.wrap(23.45)) == 23.45
-        assert PyFloat_AS_DOUBLE(space, space.wrap(23.45)) == 23.45
-        with pytest.raises(OperationError):
-            PyFloat_AsDouble(space, space.w_None)
-
-    def test_coerce(self, space):
-        assert space.type(PyNumber_Float(space, space.wrap(3))) is space.w_float
-        assert space.type(PyNumber_Float(space, space.wrap("3"))) is space.w_float
-
-        w_obj = space.appexec([], """():
-            class Coerce(object):
-                def __float__(self):
-                    return 42.5
-            return Coerce()""")
-        assert space.eq_w(PyNumber_Float(space, w_obj), space.wrap(42.5))
-
-    def test_unpack(self, space):
-        with rffi.scoped_str2charp("\x9a\x99\x99?") as ptr:
-            assert abs(_PyFloat_Unpack4(space, ptr, 1) - 1.2) < 1e-7
-        with rffi.scoped_str2charp("?\x99\x99\x9a") as ptr:
-            assert abs(_PyFloat_Unpack4(space, ptr, 0) - 1.2) < 1e-7
-        with rffi.scoped_str2charp("\x1f\x85\xebQ\xb8\x1e\t@") as ptr:
-            assert abs(_PyFloat_Unpack8(space, ptr, 1) - 3.14) < 1e-15
-        with rffi.scoped_str2charp("@\t\x1e\xb8Q\xeb\x85\x1f") as ptr:
-            assert abs(_PyFloat_Unpack8(space, ptr, 0) - 3.14) < 1e-15
-
-class AppTestFloatObject(AppTestCpythonExtensionBase):
-    def test_fromstring(self):
-        module = self.import_extension('foo', [
-            ("from_string", "METH_NOARGS",
-             """
-                 PyObject* str = PyUnicode_FromString("1234.56");
-                 PyObject* res = PyFloat_FromString(str);
-                 Py_DECREF(str);
-                 return res;
-             """),
-            ])
-        assert module.from_string() == 1234.56
-        assert type(module.from_string()) is float
-
-class AppTestFloatMacros(AppTestCpythonExtensionBase):
-    def test_return_nan(self):
-        import math
-
-        module = self.import_extension('foo', [
-            ("return_nan", "METH_NOARGS",
-             "Py_RETURN_NAN;"),
-            ])
-        assert math.isnan(module.return_nan())
-
-    def test_return_inf(self):
-        import math
-
-        module = self.import_extension('foo', [
-            ("return_inf", "METH_NOARGS",
-             "Py_RETURN_INF(10);"),
-            ])
-        inf = module.return_inf()
-        assert inf > 0
-        assert math.isinf(inf)
-
-    def test_return_inf_negative(self):
-        import math
-
-        module = self.import_extension('foo', [
-            ("return_neginf", "METH_NOARGS",
-             "Py_RETURN_INF(-10);"),
-            ])
-        neginf = module.return_neginf()
-        assert neginf < 0
-        assert math.isinf(neginf)
-
-    def test_macro_accepts_wrong_pointer_type(self):
-        module = self.import_extension('foo', [
-            ("test_macros", "METH_NOARGS",
-             """
-             PyObject* o = PyFloat_FromDouble(1.0);
-             // no PyFloatObject
-             char* dumb_pointer = (char*)o;
-
-             PyFloat_AS_DOUBLE(o);
-             PyFloat_AS_DOUBLE(dumb_pointer);
-
-             Py_RETURN_NONE;"""),
-            ])
-
-    def test_PyFloat_Check(self):
-        module = self.import_extension('foo', [
-            ("test", "METH_NOARGS",
-             """
-             PyObject* pyobj = PyFloat_FromDouble(1.0);
-             PyFloatObject* pfo = (PyFloatObject*)pyobj;
-             int res = (PyFloat_Check(pyobj) +
-                        PyFloat_CheckExact(pyobj) * 10 +
-                        PyFloat_Check(pfo) * 100 +
-                        PyFloat_CheckExact(pfo) * 1000);
-             Py_DecRef(pyobj);
-             return PyLong_FromLong(res);"""),
-            ])
-        assert module.test() == 1111
diff --git a/pypy/module/cpyext/test0/__init__.py b/pypy/module/cpyext/test0/__init__.py
new file mode 100644
diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test0/conftest.py
copy from pypy/module/cpyext/test/conftest.py
copy to pypy/module/cpyext/test0/conftest.py
diff --git a/pypy/module/cpyext/test0/test_abstract.py b/pypy/module/cpyext/test0/test_abstract.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test0/test_abstract.py
@@ -0,0 +1,130 @@
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+import pytest
+
+class AppTestBufferProtocol(AppTestCpythonExtensionBase):
+    """Tests for the old buffer protocol."""
+
+    def w_get_buffer_support(self):
+        return self.import_extension('buffer_support', [
+            ("charbuffer_as_string", "METH_O",
+             """
+                 char *ptr;
+                 Py_ssize_t size;
+                 if (PyObject_AsCharBuffer(args, (const char **)&ptr, &size) < 0)
+                     return NULL;
+                 return PyString_FromStringAndSize(ptr, size);
+             """),
+            ("check_readbuffer", "METH_O",
+             """
+                 return PyBool_FromLong(PyObject_CheckReadBuffer(args));
+             """),
+            ("readbuffer_as_string", "METH_O",
+             """
+                 const void *ptr;
+                 Py_ssize_t size;
+                 if (PyObject_AsReadBuffer(args, &ptr, &size) < 0)
+                     return NULL;
+                 return PyString_FromStringAndSize((char*)ptr, size);
+             """),
+            ("writebuffer_as_string", "METH_O",
+             """
+                 void *ptr;
+                 Py_ssize_t size;
+                 if (PyObject_AsWriteBuffer(args, &ptr, &size) < 0)
+                     return NULL;
+                 return PyString_FromStringAndSize((char*)ptr, size);
+             """),
+            ("zero_out_writebuffer", "METH_O",
+             """
+                 void *ptr;
+                 Py_ssize_t size;
+                 Py_ssize_t i;
+                 if (PyObject_AsWriteBuffer(args, &ptr, &size) < 0)
+                     return NULL;
+                 for (i = 0; i < size; i++) {
+                     ((char*)ptr)[i] = 0;
+                 }
+                 Py_RETURN_NONE;
+             """),
+            ])
+
+    def test_string(self):
+        buffer_support = self.get_buffer_support()
+
+        s = 'a\0x'
+
+        assert buffer_support.check_readbuffer(s)
+        assert s == buffer_support.readbuffer_as_string(s)
+        assert raises(TypeError, buffer_support.writebuffer_as_string, s)
+        assert s == buffer_support.charbuffer_as_string(s)
+
+    def test_buffer(self):
+        buffer_support = self.get_buffer_support()
+
+        s = 'a\0x'
+        buf = buffer(s)
+
+        assert buffer_support.check_readbuffer(buf)
+        assert s == buffer_support.readbuffer_as_string(buf)
+        assert raises(TypeError, buffer_support.writebuffer_as_string, buf)
+        assert s == buffer_support.charbuffer_as_string(buf)
+
+    def test_mmap(self):
+        import mmap
+        buffer_support = self.get_buffer_support()
+
+        s = 'a\0x'
+        mm = mmap.mmap(-1, 3)
+        mm[:] = s
+
+        assert buffer_support.check_readbuffer(mm)
+        assert s == buffer_support.readbuffer_as_string(mm)
+        assert s == buffer_support.writebuffer_as_string(mm)
+        assert s == buffer_support.charbuffer_as_string(mm)
+
+        s = '\0' * 3
+        buffer_support.zero_out_writebuffer(mm)
+        assert s == ''.join(mm)
+        assert s == buffer_support.readbuffer_as_string(mm)
+        assert s == buffer_support.writebuffer_as_string(mm)
+        assert s == buffer_support.charbuffer_as_string(mm)
+
+        s = '\0' * 3
+        ro_mm = mmap.mmap(-1, 3, access=mmap.ACCESS_READ)
+        assert buffer_support.check_readbuffer(ro_mm)
+        assert s == buffer_support.readbuffer_as_string(ro_mm)
+        assert raises(TypeError, buffer_support.writebuffer_as_string, ro_mm)
+        assert s == buffer_support.charbuffer_as_string(ro_mm)
+
+    def test_array(self):
+        import array
+        buffer_support = self.get_buffer_support()
+
+        s = 'a\0x'
+        a = array.array('B', [5, 0, 10])
+
+        buffer_support.zero_out_writebuffer(a)
+        assert list(a) == [0, 0, 0]
+
+    def test_nonbuffer(self):
+        # e.g. int
+        buffer_support = self.get_buffer_support()
+
+        assert not buffer_support.check_readbuffer(42)
+        assert raises(TypeError, buffer_support.readbuffer_as_string, 42)
+        assert raises(TypeError, buffer_support.writebuffer_as_string, 42)
+        assert raises(TypeError, buffer_support.charbuffer_as_string, 42)
+
+    def test_user_class(self):
+        class MyBuf(str):
+            pass
+        s = 'a\0x'
+        buf = MyBuf(s)
+        buffer_support = self.get_buffer_support()
+
+        assert buffer_support.check_readbuffer(buf)
+        assert s == buffer_support.readbuffer_as_string(buf)
+        assert raises(TypeError, buffer_support.writebuffer_as_string, buf)
+        assert s == buffer_support.charbuffer_as_string(buf)
+
+
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test0/test_arraymodule.py
rename from pypy/module/cpyext/test/test_arraymodule.py
rename to pypy/module/cpyext/test0/test_arraymodule.py
diff --git a/pypy/module/cpyext/test/test_boolobject.py b/pypy/module/cpyext/test0/test_boolobject.py
rename from pypy/module/cpyext/test/test_boolobject.py
rename to pypy/module/cpyext/test0/test_boolobject.py
diff --git a/pypy/module/cpyext/test/test_borrow.py b/pypy/module/cpyext/test0/test_borrow.py
rename from pypy/module/cpyext/test/test_borrow.py
rename to pypy/module/cpyext/test0/test_borrow.py
diff --git a/pypy/module/cpyext/test0/test_bufferobject.py b/pypy/module/cpyext/test0/test_bufferobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test0/test_bufferobject.py
@@ -0,0 +1,123 @@
+from rpython.rtyper.lltypesystem import lltype
+from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+from pypy.module.cpyext.api import PyObject
+
+class AppTestBufferObject(AppTestCpythonExtensionBase):
+
+    def test_FromMemory(self):
+        module = self.import_extension('foo', [
+            ("get_FromMemory", "METH_NOARGS",
+             """
+                 cbuf = malloc(4);
+                 cbuf[0] = 'a';
+                 cbuf[1] = 'b';
+                 cbuf[2] = 'c';
+                 cbuf[3] = '\\0';
+                 return PyBuffer_FromMemory(cbuf, 4);
+             """),
+            ("free_buffer", "METH_NOARGS",
+             """
+                 free(cbuf);
+                 Py_RETURN_NONE;
+             """),
+            ("check_ascharbuffer", "METH_O",
+             """
+                 char *ptr;
+                 Py_ssize_t size;
+                 if (PyObject_AsCharBuffer(args, (const char **)&ptr, &size) < 0)
+                     return NULL;
+                 return PyString_FromStringAndSize(ptr, size);
+             """)
+            ], prologue = """
+            static char* cbuf = NULL;
+            """)
+        buf = module.get_FromMemory()
+        assert str(buf) == 'abc\0'
+
+        assert module.check_ascharbuffer(buf) == 'abc\0'
+
+        module.free_buffer()
+
+    def test_Buffer_New(self):
+        module = self.import_extension('foo', [
+            ("buffer_new", "METH_NOARGS",
+             """
+                 return PyBuffer_New(150);
+             """),
+            ])
+        b = module.buffer_new()
+        raises(AttributeError, getattr, b, 'x')
+
+    def test_array_buffer(self):
+        if self.runappdirect:
+            skip('PyBufferObject not available outside buffer object.c')
+        module = self.import_extension('foo', [
+            ("roundtrip", "METH_O",
+             """
+                 PyBufferObject *buf = (PyBufferObject *)args;
+                 return PyString_FromStringAndSize(buf->b_ptr, buf->b_size);
+             """),
+            ])
+        import array
+        a = array.array('c', 'text')
+        b = buffer(a)
+        assert module.roundtrip(b) == 'text'
+
+
+    def test_issue2752(self):
+        iterations = 10
+        if self.runappdirect:
+            iterations = 2000
+        module = self.import_extension('foo', [
+            ("test_mod", 'METH_VARARGS',
+            """
+                PyObject *obj;
+                Py_buffer bp;
+                if (!PyArg_ParseTuple(args, "O", &obj))
+                    return NULL;
+
+                if (PyObject_GetBuffer(obj, &bp, PyBUF_SIMPLE) == -1)
+                    return NULL;
+                
+                if (((unsigned char*)bp.buf)[0] != '0') {
+                    void * buf = (void*)bp.buf;
+                    unsigned char val[4];
+                    char * s = PyString_AsString(obj);
+                    memcpy(val, bp.buf, 4);
+                    PyBuffer_Release(&bp);
+                    if (PyObject_GetBuffer(obj, &bp, PyBUF_SIMPLE) == -1)
+                        return NULL;
+                    PyErr_Format(PyExc_ValueError,
+                            "mismatch: %p [%x %x %x %x...] now %p [%x %x %x %x...] as str '%s'",
+                            buf, val[0], val[1], val[2], val[3],
+                            (void *)bp.buf,
+                            ((unsigned char*)bp.buf)[0],
+                            ((unsigned char*)bp.buf)[1],
+                            ((unsigned char*)bp.buf)[2],
+                            ((unsigned char*)bp.buf)[3],
+                            s);
+                    PyBuffer_Release(&bp);
+                    return NULL;
+                }
+
+                PyBuffer_Release(&bp);
+                Py_RETURN_NONE;
+            """),
+            ])
+        bufsize = 4096
+        def getdata(bufsize):
+            data = b'01234567'
+            for x in range(18):
+                data += data
+                if len(data) >= bufsize:
+                    break
+            return data
+        for j in range(iterations):
+            block = getdata(bufsize)
+            assert block[:8] == '01234567'
+            try:
+                module.test_mod(block)
+            except ValueError as e:
+                print("%s at it=%d" % (e, j))
+                assert False
diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test0/test_bytearrayobject.py
rename from pypy/module/cpyext/test/test_bytearrayobject.py
rename to pypy/module/cpyext/test0/test_bytearrayobject.py
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test0/test_bytesobject.py
rename from pypy/module/cpyext/test/test_bytesobject.py
rename to pypy/module/cpyext/test0/test_bytesobject.py
diff --git a/pypy/module/cpyext/test/test_capsule.py b/pypy/module/cpyext/test0/test_capsule.py
rename from pypy/module/cpyext/test/test_capsule.py
rename to pypy/module/cpyext/test0/test_capsule.py
diff --git a/pypy/module/cpyext/test/test_cell.py b/pypy/module/cpyext/test0/test_cell.py
rename from pypy/module/cpyext/test/test_cell.py
rename to pypy/module/cpyext/test0/test_cell.py
diff --git a/pypy/module/cpyext/test/test_classobject.py b/pypy/module/cpyext/test0/test_classobject.py
rename from pypy/module/cpyext/test/test_classobject.py
rename to pypy/module/cpyext/test0/test_classobject.py
diff --git a/pypy/module/cpyext/test/test_codecs.py b/pypy/module/cpyext/test0/test_codecs.py
rename from pypy/module/cpyext/test/test_codecs.py
rename to pypy/module/cpyext/test0/test_codecs.py
diff --git a/pypy/module/cpyext/test/test_complexobject.py b/pypy/module/cpyext/test0/test_complexobject.py
rename from pypy/module/cpyext/test/test_complexobject.py
rename to pypy/module/cpyext/test0/test_complexobject.py
diff --git a/pypy/module/cpyext/test/test_cparser.py b/pypy/module/cpyext/test0/test_cparser.py
rename from pypy/module/cpyext/test/test_cparser.py
rename to pypy/module/cpyext/test0/test_cparser.py
diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test0/test_datetime.py
rename from pypy/module/cpyext/test/test_datetime.py
rename to pypy/module/cpyext/test0/test_datetime.py
diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test0/test_dictobject.py
rename from pypy/module/cpyext/test/test_dictobject.py
rename to pypy/module/cpyext/test0/test_dictobject.py
diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test0/test_eval.py
rename from pypy/module/cpyext/test/test_eval.py
rename to pypy/module/cpyext/test0/test_eval.py
diff --git a/pypy/module/cpyext/test/test_fileobject.py b/pypy/module/cpyext/test0/test_fileobject.py
rename from pypy/module/cpyext/test/test_fileobject.py
rename to pypy/module/cpyext/test0/test_fileobject.py
diff --git a/pypy/module/cpyext/test0/test_floatobject.py b/pypy/module/cpyext/test0/test_floatobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test0/test_floatobject.py
@@ -0,0 +1,196 @@
+import pytest
+from pypy.interpreter.error import OperationError
+from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+from rpython.rtyper.lltypesystem import rffi
+from pypy.module.cpyext.floatobject import (
+    PyFloat_FromDouble, PyFloat_AsDouble, PyFloat_AS_DOUBLE, PyNumber_Float,
+    _PyFloat_Unpack4, _PyFloat_Unpack8)
+
+class TestFloatObject(BaseApiTest):
+    def test_floatobject(self, space):
+        assert space.unwrap(PyFloat_FromDouble(space, 3.14)) == 3.14
+        assert PyFloat_AsDouble(space, space.wrap(23.45)) == 23.45
+        assert PyFloat_AS_DOUBLE(space, space.wrap(23.45)) == 23.45
+        with pytest.raises(OperationError):
+            PyFloat_AsDouble(space, space.w_None)
+
+    def test_coerce(self, space):
+        assert space.type(PyNumber_Float(space, space.wrap(3))) is space.w_float
+        assert space.type(PyNumber_Float(space, space.wrap("3"))) is space.w_float
+
+        w_obj = space.appexec([], """():
+            class Coerce(object):
+                def __float__(self):
+                    return 42.5
+            return Coerce()""")
+        assert space.eq_w(PyNumber_Float(space, w_obj), space.wrap(42.5))
+
+    def test_unpack(self, space):
+        with rffi.scoped_str2charp("\x9a\x99\x99?") as ptr:
+            assert abs(_PyFloat_Unpack4(space, ptr, 1) - 1.2) < 1e-7
+        with rffi.scoped_str2charp("?\x99\x99\x9a") as ptr:
+            assert abs(_PyFloat_Unpack4(space, ptr, 0) - 1.2) < 1e-7
+        with rffi.scoped_str2charp("\x1f\x85\xebQ\xb8\x1e\t@") as ptr:
+            assert abs(_PyFloat_Unpack8(space, ptr, 1) - 3.14) < 1e-15
+        with rffi.scoped_str2charp("@\t\x1e\xb8Q\xeb\x85\x1f") as ptr:
+            assert abs(_PyFloat_Unpack8(space, ptr, 0) - 3.14) < 1e-15
+
+class AppTestFloatObject(AppTestCpythonExtensionBase):
+    def test_fromstring(self):
+        module = self.import_extension('foo', [
+            ("from_string", "METH_NOARGS",
+             """
+                 PyObject* str = PyUnicode_FromString("1234.56");
+                 PyObject* res = PyFloat_FromString(str);
+                 Py_DECREF(str);
+                 return res;
+             """),
+            ])
+        assert module.from_string() == 1234.56
+        assert type(module.from_string()) is float
+
+class AppTestFloatMacros(AppTestCpythonExtensionBase):
+    def test_return_nan(self):
+        import math
+
+        module = self.import_extension('foo', [
+            ("return_nan", "METH_NOARGS",
+             "Py_RETURN_NAN;"),
+            ])
+        assert math.isnan(module.return_nan())
+
+    def test_return_inf(self):
+        import math
+
+        module = self.import_extension('foo', [
+            ("return_inf", "METH_NOARGS",
+             "Py_RETURN_INF(10);"),
+            ])
+        inf = module.return_inf()
+        assert inf > 0
+        assert math.isinf(inf)
+
+    def test_return_inf_negative(self):
+        import math
+
+        module = self.import_extension('foo', [
+            ("return_neginf", "METH_NOARGS",
+             "Py_RETURN_INF(-10);"),
+            ])
+        neginf = module.return_neginf()
+        assert neginf < 0
+        assert math.isinf(neginf)
+
+    def test_macro_accepts_wrong_pointer_type(self):
+        module = self.import_extension('foo', [
+            ("test_macros", "METH_NOARGS",
+             """
+             PyObject* o = PyFloat_FromDouble(1.0);
+             // no PyFloatObject
+             char* dumb_pointer = (char*)o;
+
+             PyFloat_AS_DOUBLE(o);
+             PyFloat_AS_DOUBLE(dumb_pointer);
+
+             Py_RETURN_NONE;"""),
+            ])
+
+    def test_PyFloat_Check(self):
+        module = self.import_extension('foo', [
+            ("test", "METH_NOARGS",
+             """
+             PyObject* pyobj = PyFloat_FromDouble(1.0);
+             PyFloatObject* pfo = (PyFloatObject*)pyobj;
+             int res = (PyFloat_Check(pyobj) +
+                        PyFloat_CheckExact(pyobj) * 10 +
+                        PyFloat_Check(pfo) * 100 +
+                        PyFloat_CheckExact(pfo) * 1000);
+             Py_DecRef(pyobj);
+             return PyLong_FromLong(res);"""),
+            ])
+        assert module.test() == 1111
+
+    def test_pymath_consts(self):
+        # test preprocessor constants in their string form to avoid
+        # floating-point conversion issues (and to avoid having to
+        # conditionalize on compiler support for long double)
+        for const_name, const_strval in [
+                ('Py_MATH_PIl', "3.1415926535897932384626433832795029L"),
+                ('Py_MATH_PI', "3.14159265358979323846"),
+                ('Py_MATH_El', "2.7182818284590452353602874713526625L"),
+                ('Py_MATH_E', "2.7182818284590452354"),
+                ('Py_MATH_TAU', "6.2831853071795864769252867665590057683943L"),
+            ]:
+            module = self.import_extension('foo_%s' % const_name, [
+                ("test", "METH_NOARGS",
+                 """
+                 #define xstr(s) str(s)
+                 #define str(s) #s
+                 return PyString_FromString(xstr(%s));""" % const_name)
+            ])
+            assert module.test() == const_strval
+
+    def test_Py_IS_NAN(self):
+        module = self.import_extension('foo', [
+            ("test", "METH_O",
+             """
+                 double d = PyFloat_AsDouble(args);
+                 return PyBool_FromLong(Py_IS_NAN(d));
+             """),
+            ])
+        assert not module.test(0)
+        assert not module.test(1)
+        assert not module.test(-1)
+        assert not module.test(float('inf'))
+        assert module.test(float('nan'))
+
+    def test_Py_IS_INFINITY(self):
+        module = self.import_extension('foo', [
+            ("test", "METH_O",
+             """
+                 double d = PyFloat_AsDouble(args);
+                 return PyBool_FromLong(Py_IS_INFINITY(d));
+             """),
+            ])
+        assert not module.test(0)
+        assert not module.test(1)
+        assert not module.test(-1)
+        assert not module.test(float('nan'))
+        assert module.test(float('inf'))
+        assert module.test(float('-inf'))
+
+    def test_Py_IS_FINITE(self):
+        module = self.import_extension('foo', [
+            ("test", "METH_O",
+             """
+                 double d = PyFloat_AsDouble(args);
+                 return PyBool_FromLong(Py_IS_FINITE(d));
+             """),
+            ])
+        assert module.test(0)
+        assert module.test(1)
+        assert module.test(-1)
+        assert not module.test(float('nan'))
+        assert not module.test(float('inf'))
+        assert not module.test(float('-inf'))
+
+    def test_Py_HUGE_VAL(self):
+        module = self.import_extension('foo', [
+            ("test", "METH_NOARGS",
+             """
+                 return PyFloat_FromDouble(Py_HUGE_VAL);
+             """),
+            ])
+        assert module.test() == float('inf')
+
+    def test_Py_NAN(self):
+        module = self.import_extension('foo', [
+            ("test", "METH_NOARGS",
+             """
+                 return PyFloat_FromDouble(Py_NAN);
+             """),
+            ])
+        import struct
+        float_bits = struct.Struct('d').pack
+        assert float_bits(module.test()) == float_bits(float('nan'))
diff --git a/pypy/module/cpyext/test/test_frameobject.py b/pypy/module/cpyext/test0/test_frameobject.py
rename from pypy/module/cpyext/test/test_frameobject.py
rename to pypy/module/cpyext/test0/test_frameobject.py
diff --git a/pypy/module/cpyext/test/test_funcobject.py b/pypy/module/cpyext/test0/test_funcobject.py
rename from pypy/module/cpyext/test/test_funcobject.py
rename to pypy/module/cpyext/test0/test_funcobject.py
diff --git a/pypy/module/cpyext/test/test_genobject.py b/pypy/module/cpyext/test0/test_genobject.py
rename from pypy/module/cpyext/test/test_genobject.py
rename to pypy/module/cpyext/test0/test_genobject.py
diff --git a/pypy/module/cpyext/test/test_getargs.py b/pypy/module/cpyext/test0/test_getargs.py
rename from pypy/module/cpyext/test/test_getargs.py
rename to pypy/module/cpyext/test0/test_getargs.py
diff --git a/pypy/module/cpyext/test/test_import.py b/pypy/module/cpyext/test0/test_import.py
rename from pypy/module/cpyext/test/test_import.py
rename to pypy/module/cpyext/test0/test_import.py
diff --git a/pypy/module/cpyext/test0/test_intobject.py b/pypy/module/cpyext/test0/test_intobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test0/test_intobject.py
@@ -0,0 +1,247 @@
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+from pypy.module.cpyext.intobject import (
+    PyInt_Check, PyInt_AsLong, PyInt_AS_LONG,
+    PyInt_AsUnsignedLong, PyInt_AsUnsignedLongMask,
+    PyInt_AsUnsignedLongLongMask)
+from pypy.module.cpyext.pyobject import (decref, make_ref,
+                                         get_w_obj_and_decref)
+from pypy.module.cpyext.state import State
+import sys
+
+class TestIntObject(BaseApiTest):
+    def test_intobject(self, space):
+        state = space.fromcache(State)
+        assert PyInt_Check(space, space.wrap(3))
+        assert PyInt_Check(space, space.w_True)
+        assert not PyInt_Check(space, space.wrap((1, 2, 3)))
+        for i in [3, -5, -1, -sys.maxint, sys.maxint - 1]:
+            x = PyInt_AsLong(space, space.wrap(i))
+            y = PyInt_AS_LONG(space, space.wrap(i))
+            assert x == i
+            assert y == i
+            py_x = state.C.PyInt_FromLong(x + 1)
+            w_x = get_w_obj_and_decref(space, py_x)
+            assert space.type(w_x) is space.w_int
+            assert space.eq_w(w_x, space.wrap(i + 1))
+
+        with raises_w(space, TypeError):
+            PyInt_AsLong(space, space.w_None)
+
+        with raises_w(space, TypeError):
+            PyInt_AsLong(space, None)
+
+        assert PyInt_AsUnsignedLong(space, space.wrap(sys.maxint)) == sys.maxint
+        with raises_w(space, ValueError):
+            PyInt_AsUnsignedLong(space, space.wrap(-5))
+
+        assert (PyInt_AsUnsignedLongMask(space, space.wrap(sys.maxint))
+                == sys.maxint)
+        assert (PyInt_AsUnsignedLongMask(space, space.wrap(10 ** 30))
+                == 10 ** 30 % ((sys.maxint + 1) * 2))
+
+        assert (PyInt_AsUnsignedLongLongMask(space, space.wrap(sys.maxint))
+                == sys.maxint)
+        assert (PyInt_AsUnsignedLongLongMask(space, space.wrap(10 ** 30))
+                == 10 ** 30 % (2 ** 64))
+
+    def test_freelist_direct(self, space):
+        state = space.fromcache(State)
+        p_x = state.C.PyInt_FromLong(12345678)
+        decref(space, p_x)
+        p_y = state.C.PyInt_FromLong(87654321)
+        # check that the address is the same, i.e. that the freelist did its
+        # job
+        assert p_x == p_y
+        decref(space, p_y)
+
+    def test_freelist_make_ref(self, space):
+        w_x = space.newint(12345678)
+        w_y = space.newint(87654321)
+        p_x = make_ref(space, w_x)
+        decref(space, p_x)
+        p_y = make_ref(space, w_y)
+        # check that the address is the same: note that w_x does NOT keep p_x
+        # alive, because in make_ref we have a special case for ints
+        assert p_x == p_y
+        decref(space, p_y)
+
+    def test_freelist_int_subclass(self, space):
+        w_MyInt = space.appexec([], """():
+            class MyInt(int):
+                pass
+            return MyInt""")
+        w_x = space.call_function(w_MyInt, space.newint(12345678))
+        w_y = space.call_function(w_MyInt, space.newint(87654321))
+        p_x = make_ref(space, w_x)
+        decref(space, p_x)
+        p_y = make_ref(space, w_y)
+        # now the address is different because the freelist does not work for
+        # int subclasses
+        assert p_x != p_y
+        decref(space, p_y)
+
+    def test_coerce(self, space):
+        w_obj = space.appexec([], """():
+            class Coerce(object):
+                def __int__(self):
+                    return 42
+            return Coerce()""")
+        assert PyInt_AsLong(space, w_obj) == 42
+
+class AppTestIntObject(AppTestCpythonExtensionBase):
+    def test_fromstring(self):
+        module = self.import_extension('foo', [
+            ("from_string", "METH_NOARGS",
+             """
+                 return PyInt_FromString("1234", NULL, 16);
+             """),
+            ])
+        assert module.from_string() == 0x1234
+        assert type(module.from_string()) is int
+
+    def test_size_t(self):
+        module = self.import_extension('foo', [
+            ("values", "METH_NOARGS",
+             """
+                 return Py_BuildValue("NNNN",
+                     PyInt_FromSize_t(123),
+                     PyInt_FromSize_t((size_t)-1),
+                     PyInt_FromSsize_t(123),
+                     PyInt_FromSsize_t((size_t)-1));
+             """),
+            ])
+        values = module.values()
+        types = [type(x) for x in values]
+        assert types == [int, long, int, int]
+
+    def test_int_subtype(self):
+        module = self.import_extension(
+            'foo', [
+            ("newEnum", "METH_VARARGS",
+             """
+                EnumObject *enumObj;
+                int intval;
+                PyObject *name;
+
+                if (!PyArg_ParseTuple(args, "Oi", &name, &intval))
+                    return NULL;
+
+                enumObj = PyObject_New(EnumObject, &Enum_Type);
+                if (!enumObj) {
+                    return NULL;
+                }
+
+                enumObj->ob_ival = intval;
+                Py_INCREF(name);
+                enumObj->ob_name = name;
+
+                return (PyObject *)enumObj;
+             """),
+            ],
+            prologue="""
+            #include "structmember.h"
+            typedef struct
+            {
+                PyObject_HEAD
+                long ob_ival;
+                PyObject* ob_name;
+            } EnumObject;
+
+            static void
+            enum_dealloc(PyObject *op)
+            {
+                    Py_DECREF(((EnumObject *)op)->ob_name);
+                    Py_TYPE(op)->tp_free(op);
+            }
+
+            static PyMemberDef enum_members[] = {
+                {"name", T_OBJECT, offsetof(EnumObject, ob_name), 0, NULL},
+                {NULL}  /* Sentinel */
+            };
+
+            PyTypeObject Enum_Type = {
+                PyVarObject_HEAD_INIT(NULL, 0)
+                /*tp_name*/             "Enum",
+                /*tp_basicsize*/        sizeof(EnumObject),
+                /*tp_itemsize*/         0,
+                /*tp_dealloc*/          enum_dealloc,
+                /*tp_print*/            0,
+                /*tp_getattr*/          0,
+                /*tp_setattr*/          0,
+                /*tp_compare*/          0,
+                /*tp_repr*/             0,
+                /*tp_as_number*/        0,
+                /*tp_as_sequence*/      0,
+                /*tp_as_mapping*/       0,
+                /*tp_hash*/             0,
+                /*tp_call*/             0,
+                /*tp_str*/              0,
+                /*tp_getattro*/         0,
+                /*tp_setattro*/         0,
+                /*tp_as_buffer*/        0,
+                /*tp_flags*/            Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,
+                /*tp_doc*/              0,
+                /*tp_traverse*/         0,
+                /*tp_clear*/            0,
+                /*tp_richcompare*/      0,
+                /*tp_weaklistoffset*/   0,
+                /*tp_iter*/             0,
+                /*tp_iternext*/         0,
+                /*tp_methods*/          0,
+                /*tp_members*/          enum_members,
+                /*tp_getset*/           0,
+                /*tp_base*/             0, /* set to &PyInt_Type in init function for MSVC */
+                /*tp_dict*/             0,
+                /*tp_descr_get*/        0,
+                /*tp_descr_set*/        0,
+                /*tp_dictoffset*/       0,
+                /*tp_init*/             0,
+                /*tp_alloc*/            0,
+                /*tp_new*/              0
+            };
+            """, more_init = '''
+                Enum_Type.tp_base = &PyInt_Type;
+                if (PyType_Ready(&Enum_Type) < 0) INITERROR;
+            ''')
+
+        a = module.newEnum("ULTIMATE_ANSWER", 42)
+        assert type(a).__name__ == "Enum"
+        assert isinstance(a, int)
+        assert a == int(a) == 42
+        assert a.name == "ULTIMATE_ANSWER"
+
+    def test_int_cast(self):
+        mod = self.import_extension('foo', [
+                #prove it works for ints
+                ("test_int", "METH_NOARGS",
+                """
+                PyObject * obj = PyInt_FromLong(42);
+                PyObject * val;
+                if (!PyInt_Check(obj)) {
+                    Py_DECREF(obj);
+                    PyErr_SetNone(PyExc_ValueError);
+                    return NULL;
+                }
+                val = PyInt_FromLong(((PyIntObject *)obj)->ob_ival);
+                Py_DECREF(obj);
+                return val;
+                """
+                ),
+                ])
+        i = mod.test_int()
+        assert isinstance(i, int)
+        assert i == 42
+
+    def test_int_macros(self):
+        mod = self.import_extension('foo', [
+                ("test_macros", "METH_NOARGS",
+                """
+                PyObject * obj = PyInt_FromLong(42);
+                PyIntObject * i = (PyIntObject*)obj;
+                PyInt_AS_LONG(obj);
+                PyInt_AS_LONG(i);
+                Py_RETURN_NONE;
+                """
+                ),
+                ])
diff --git a/pypy/module/cpyext/test/test_iterator.py b/pypy/module/cpyext/test0/test_iterator.py
rename from pypy/module/cpyext/test/test_iterator.py
rename to pypy/module/cpyext/test0/test_iterator.py
diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test0/test_listobject.py
rename from pypy/module/cpyext/test/test_listobject.py
rename to pypy/module/cpyext/test0/test_listobject.py
diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test0/test_longobject.py
rename from pypy/module/cpyext/test/test_longobject.py
rename to pypy/module/cpyext/test0/test_longobject.py
diff --git a/pypy/module/cpyext/test/test_mapping.py b/pypy/module/cpyext/test0/test_mapping.py
rename from pypy/module/cpyext/test/test_mapping.py
rename to pypy/module/cpyext/test0/test_mapping.py
diff --git a/pypy/module/cpyext/test/test_marshal.py b/pypy/module/cpyext/test0/test_marshal.py
rename from pypy/module/cpyext/test/test_marshal.py
rename to pypy/module/cpyext/test0/test_marshal.py
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test0/test_memoryobject.py
rename from pypy/module/cpyext/test/test_memoryobject.py
rename to pypy/module/cpyext/test0/test_memoryobject.py
diff --git a/pypy/module/cpyext/test/test_methodobject.py b/pypy/module/cpyext/test0/test_methodobject.py
rename from pypy/module/cpyext/test/test_methodobject.py
rename to pypy/module/cpyext/test0/test_methodobject.py
diff --git a/pypy/module/cpyext/test/test_module.py b/pypy/module/cpyext/test0/test_module.py
rename from pypy/module/cpyext/test/test_module.py
rename to pypy/module/cpyext/test0/test_module.py
diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test0/test_ndarrayobject.py
rename from pypy/module/cpyext/test/test_ndarrayobject.py
rename to pypy/module/cpyext/test0/test_ndarrayobject.py
diff --git a/pypy/module/cpyext/test/test_number.py b/pypy/module/cpyext/test0/test_number.py
rename from pypy/module/cpyext/test/test_number.py
rename to pypy/module/cpyext/test0/test_number.py
diff --git a/pypy/module/cpyext/test1/__init__.py b/pypy/module/cpyext/test1/__init__.py
new file mode 100644
diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test1/conftest.py
copy from pypy/module/cpyext/test/conftest.py
copy to pypy/module/cpyext/test1/conftest.py
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test1/test_object.py
rename from pypy/module/cpyext/test/test_object.py
rename to pypy/module/cpyext/test1/test_object.py
diff --git a/pypy/module/cpyext/test1/test_pycobject.py b/pypy/module/cpyext/test1/test_pycobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test1/test_pycobject.py
@@ -0,0 +1,30 @@
+import py
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+
+class AppTestStringObject(AppTestCpythonExtensionBase):
+    def test_pycobject_import(self):
+        module = self.import_extension('foo', [
+            ("set_ptr", "METH_O",
+             """
+                 PyObject *pointer, *module;
+                 void *ptr = PyLong_AsVoidPtr(args);
+                 if (PyErr_Occurred()) return NULL;
+                 pointer = PyCObject_FromVoidPtr(ptr, NULL);
+                 if (PyErr_Occurred()) return NULL;
+                 module = PyImport_ImportModule("foo");
+                 PyModule_AddObject(module, "_ptr", pointer);
+                 Py_DECREF(module);
+                 if (PyErr_Occurred()) return NULL;
+                 Py_RETURN_NONE;
+             """),
+            ("get_ptr", "METH_NOARGS",
+             """
+                 void *ptr = PyCObject_Import("foo", "_ptr");
+                 if (PyErr_Occurred()) return NULL;
+                 return PyLong_FromVoidPtr(ptr);
+             """)])
+        module.set_ptr(1234)
+        assert "PyCObject object" in str(module._ptr)
+        import gc; gc.collect()
+        assert module.get_ptr() == 1234
+        del module._ptr
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test1/test_pyerrors.py
rename from pypy/module/cpyext/test/test_pyerrors.py
rename to pypy/module/cpyext/test1/test_pyerrors.py
diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test1/test_pyfile.py
rename from pypy/module/cpyext/test/test_pyfile.py
rename to pypy/module/cpyext/test1/test_pyfile.py
diff --git a/pypy/module/cpyext/test/test_pysignals.py b/pypy/module/cpyext/test1/test_pysignals.py
rename from pypy/module/cpyext/test/test_pysignals.py
rename to pypy/module/cpyext/test1/test_pysignals.py
diff --git a/pypy/module/cpyext/test/test_pystate.py b/pypy/module/cpyext/test1/test_pystate.py
rename from pypy/module/cpyext/test/test_pystate.py
rename to pypy/module/cpyext/test1/test_pystate.py
diff --git a/pypy/module/cpyext/test/test_pystrtod.py b/pypy/module/cpyext/test1/test_pystrtod.py
rename from pypy/module/cpyext/test/test_pystrtod.py
rename to pypy/module/cpyext/test1/test_pystrtod.py
diff --git a/pypy/module/cpyext/test/test_sequence.py b/pypy/module/cpyext/test1/test_sequence.py
rename from pypy/module/cpyext/test/test_sequence.py
rename to pypy/module/cpyext/test1/test_sequence.py
diff --git a/pypy/module/cpyext/test/test_setobject.py b/pypy/module/cpyext/test1/test_setobject.py
rename from pypy/module/cpyext/test/test_setobject.py
rename to pypy/module/cpyext/test1/test_setobject.py
diff --git a/pypy/module/cpyext/test/test_sliceobject.py b/pypy/module/cpyext/test1/test_sliceobject.py
rename from pypy/module/cpyext/test/test_sliceobject.py
rename to pypy/module/cpyext/test1/test_sliceobject.py
diff --git a/pypy/module/cpyext/test/test_structseq.py b/pypy/module/cpyext/test1/test_structseq.py
rename from pypy/module/cpyext/test/test_structseq.py
rename to pypy/module/cpyext/test1/test_structseq.py
diff --git a/pypy/module/cpyext/test/test_sysmodule.py b/pypy/module/cpyext/test1/test_sysmodule.py
rename from pypy/module/cpyext/test/test_sysmodule.py
rename to pypy/module/cpyext/test1/test_sysmodule.py
diff --git a/pypy/module/cpyext/test/test_thread.py b/pypy/module/cpyext/test1/test_thread.py
rename from pypy/module/cpyext/test/test_thread.py
rename to pypy/module/cpyext/test1/test_thread.py
diff --git a/pypy/module/cpyext/test/test_traceback.py b/pypy/module/cpyext/test1/test_traceback.py
rename from pypy/module/cpyext/test/test_traceback.py
rename to pypy/module/cpyext/test1/test_traceback.py
diff --git a/pypy/module/cpyext/test/test_translate.py b/pypy/module/cpyext/test1/test_translate.py
rename from pypy/module/cpyext/test/test_translate.py
rename to pypy/module/cpyext/test1/test_translate.py
diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test1/test_tupleobject.py
rename from pypy/module/cpyext/test/test_tupleobject.py
rename to pypy/module/cpyext/test1/test_tupleobject.py
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test1/test_typeobject.py
rename from pypy/module/cpyext/test/test_typeobject.py
rename to pypy/module/cpyext/test1/test_typeobject.py
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test1/test_unicodeobject.py
rename from pypy/module/cpyext/test/test_unicodeobject.py
rename to pypy/module/cpyext/test1/test_unicodeobject.py
diff --git a/pypy/module/cpyext/test/test_userslots.py b/pypy/module/cpyext/test1/test_userslots.py
rename from pypy/module/cpyext/test/test_userslots.py
rename to pypy/module/cpyext/test1/test_userslots.py
diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test1/test_version.py
rename from pypy/module/cpyext/test/test_version.py
rename to pypy/module/cpyext/test1/test_version.py
diff --git a/pypy/module/cpyext/test/test_weakref.py b/pypy/module/cpyext/test1/test_weakref.py
rename from pypy/module/cpyext/test/test_weakref.py
rename to pypy/module/cpyext/test1/test_weakref.py
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -85,6 +85,8 @@
 
     w_mod = check_sys_modules_w(space, modulename)
     if w_mod:
+    if parts[-1] == '':
+        del parts[-1]
         return w_mod
     lock = getimportlock(space)
     try:
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -302,6 +302,13 @@
         assert ambig == sys.modules.get('ambig')
         assert hasattr(ambig,'imapackage')
 
+    def test_trailing_dot(self):
+        # bug-for-bug compatibility with CPython
+        import sys
+        __import__('pkg.pkg1.')
+        assert 'pkg.pkg1' in sys.modules
+        assert 'pkg.pkg1.' not in sys.modules
+
     def test_from_a(self):
         import sys
         from a import imamodule
@@ -709,7 +716,6 @@
             exec("from pkg.withoutall import *", d)
             assert "" in d
 
-
     def test_import_star_with_bogus___all__(self):
         for case in ["not-imported-yet", "already-imported"]:
             try:
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -388,7 +388,7 @@
                 not self.flags & NPY.ARRAY_F_CONTIGUOUS):
            raise oefmt(errtype, "ndarray is not Fortran contiguous")
         if ((flags & space.BUF_ANY_CONTIGUOUS) == space.BUF_ANY_CONTIGUOUS and
-                not (self.flags & NPY.ARRAY_F_CONTIGUOUS and
+                not (self.flags & NPY.ARRAY_F_CONTIGUOUS or
                      self.flags & NPY.ARRAY_C_CONTIGUOUS)):
            raise oefmt(errtype, "ndarray is not contiguous")
         if ((flags & space.BUF_STRIDES) != space.BUF_STRIDES and
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -874,11 +874,11 @@
 
         if traits.str is unicode:
             if path and path[-1] not in (u'/', u'\\', u':'):
-                path += u'/'
+                path += u'\\'
             mask = path + u'*.*'
         else:
             if path and path[-1] not in ('/', '\\', ':'):
-                path += '/'
+                path += '\\'
             mask = path + '*.*'
 
         filedata = lltype.malloc(win32traits.WIN32_FIND_DATA, flavor='raw')

From pypy.commits at gmail.com  Wed Aug 29 03:49:36 2018
From: pypy.commits at gmail.com (mattip)
Date: Wed, 29 Aug 2018 00:49:36 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix bad automatic merge
Message-ID: <5b865010.1c69fb81.12231.5d6c@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r95038:33943b0827fd
Date: 2018-08-29 08:21 +0200
http://bitbucket.org/pypy/pypy/changeset/33943b0827fd/

Log:	fix bad automatic merge

diff --git a/pypy/module/cpyext/test0/test_abstract.py b/pypy/module/cpyext/test0/test_abstract.py
deleted file mode 100644
--- a/pypy/module/cpyext/test0/test_abstract.py
+++ /dev/null
@@ -1,130 +0,0 @@
-from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-import pytest
-
-class AppTestBufferProtocol(AppTestCpythonExtensionBase):
-    """Tests for the old buffer protocol."""
-
-    def w_get_buffer_support(self):
-        return self.import_extension('buffer_support', [
-            ("charbuffer_as_string", "METH_O",
-             """
-                 char *ptr;
-                 Py_ssize_t size;
-                 if (PyObject_AsCharBuffer(args, (const char **)&ptr, &size) < 0)
-                     return NULL;
-                 return PyString_FromStringAndSize(ptr, size);
-             """),
-            ("check_readbuffer", "METH_O",
-             """
-                 return PyBool_FromLong(PyObject_CheckReadBuffer(args));
-             """),
-            ("readbuffer_as_string", "METH_O",
-             """
-                 const void *ptr;
-                 Py_ssize_t size;
-                 if (PyObject_AsReadBuffer(args, &ptr, &size) < 0)
-                     return NULL;
-                 return PyString_FromStringAndSize((char*)ptr, size);
-             """),
-            ("writebuffer_as_string", "METH_O",
-             """
-                 void *ptr;
-                 Py_ssize_t size;
-                 if (PyObject_AsWriteBuffer(args, &ptr, &size) < 0)
-                     return NULL;
-                 return PyString_FromStringAndSize((char*)ptr, size);
-             """),
-            ("zero_out_writebuffer", "METH_O",
-             """
-                 void *ptr;
-                 Py_ssize_t size;
-                 Py_ssize_t i;
-                 if (PyObject_AsWriteBuffer(args, &ptr, &size) < 0)
-                     return NULL;
-                 for (i = 0; i < size; i++) {
-                     ((char*)ptr)[i] = 0;
-                 }
-                 Py_RETURN_NONE;
-             """),
-            ])
-
-    def test_string(self):
-        buffer_support = self.get_buffer_support()
-
-        s = 'a\0x'
-
-        assert buffer_support.check_readbuffer(s)
-        assert s == buffer_support.readbuffer_as_string(s)
-        assert raises(TypeError, buffer_support.writebuffer_as_string, s)
-        assert s == buffer_support.charbuffer_as_string(s)
-
-    def test_buffer(self):
-        buffer_support = self.get_buffer_support()
-
-        s = 'a\0x'
-        buf = buffer(s)
-
-        assert buffer_support.check_readbuffer(buf)
-        assert s == buffer_support.readbuffer_as_string(buf)
-        assert raises(TypeError, buffer_support.writebuffer_as_string, buf)
-        assert s == buffer_support.charbuffer_as_string(buf)
-
-    def test_mmap(self):
-        import mmap
-        buffer_support = self.get_buffer_support()
-
-        s = 'a\0x'
-        mm = mmap.mmap(-1, 3)
-        mm[:] = s
-
-        assert buffer_support.check_readbuffer(mm)
-        assert s == buffer_support.readbuffer_as_string(mm)
-        assert s == buffer_support.writebuffer_as_string(mm)
-        assert s == buffer_support.charbuffer_as_string(mm)
-
-        s = '\0' * 3
-        buffer_support.zero_out_writebuffer(mm)
-        assert s == ''.join(mm)
-        assert s == buffer_support.readbuffer_as_string(mm)
-        assert s == buffer_support.writebuffer_as_string(mm)
-        assert s == buffer_support.charbuffer_as_string(mm)
-
-        s = '\0' * 3
-        ro_mm = mmap.mmap(-1, 3, access=mmap.ACCESS_READ)
-        assert buffer_support.check_readbuffer(ro_mm)
-        assert s == buffer_support.readbuffer_as_string(ro_mm)
-        assert raises(TypeError, buffer_support.writebuffer_as_string, ro_mm)
-        assert s == buffer_support.charbuffer_as_string(ro_mm)
-
-    def test_array(self):
-        import array
-        buffer_support = self.get_buffer_support()
-
-        s = 'a\0x'
-        a = array.array('B', [5, 0, 10])
-
-        buffer_support.zero_out_writebuffer(a)
-        assert list(a) == [0, 0, 0]
-
-    def test_nonbuffer(self):
-        # e.g. int
-        buffer_support = self.get_buffer_support()
-
-        assert not buffer_support.check_readbuffer(42)
-        assert raises(TypeError, buffer_support.readbuffer_as_string, 42)
-        assert raises(TypeError, buffer_support.writebuffer_as_string, 42)
-        assert raises(TypeError, buffer_support.charbuffer_as_string, 42)
-
-    def test_user_class(self):
-        class MyBuf(str):
-            pass
-        s = 'a\0x'
-        buf = MyBuf(s)
-        buffer_support = self.get_buffer_support()
-
-        assert buffer_support.check_readbuffer(buf)
-        assert s == buffer_support.readbuffer_as_string(buf)
-        assert raises(TypeError, buffer_support.writebuffer_as_string, buf)
-        assert s == buffer_support.charbuffer_as_string(buf)
-
-
diff --git a/pypy/module/cpyext/test0/test_bufferobject.py b/pypy/module/cpyext/test0/test_bufferobject.py
deleted file mode 100644
--- a/pypy/module/cpyext/test0/test_bufferobject.py
+++ /dev/null
@@ -1,123 +0,0 @@
-from rpython.rtyper.lltypesystem import lltype
-from pypy.module.cpyext.test.test_api import BaseApiTest
-from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from pypy.module.cpyext.api import PyObject
-
-class AppTestBufferObject(AppTestCpythonExtensionBase):
-
-    def test_FromMemory(self):
-        module = self.import_extension('foo', [
-            ("get_FromMemory", "METH_NOARGS",
-             """
-                 cbuf = malloc(4);
-                 cbuf[0] = 'a';
-                 cbuf[1] = 'b';
-                 cbuf[2] = 'c';
-                 cbuf[3] = '\\0';
-                 return PyBuffer_FromMemory(cbuf, 4);
-             """),
-            ("free_buffer", "METH_NOARGS",
-             """
-                 free(cbuf);
-                 Py_RETURN_NONE;
-             """),
-            ("check_ascharbuffer", "METH_O",
-             """
-                 char *ptr;
-                 Py_ssize_t size;
-                 if (PyObject_AsCharBuffer(args, (const char **)&ptr, &size) < 0)
-                     return NULL;
-                 return PyString_FromStringAndSize(ptr, size);
-             """)
-            ], prologue = """
-            static char* cbuf = NULL;
-            """)
-        buf = module.get_FromMemory()
-        assert str(buf) == 'abc\0'
-
-        assert module.check_ascharbuffer(buf) == 'abc\0'
-
-        module.free_buffer()
-
-    def test_Buffer_New(self):
-        module = self.import_extension('foo', [
-            ("buffer_new", "METH_NOARGS",
-             """
-                 return PyBuffer_New(150);
-             """),
-            ])
-        b = module.buffer_new()
-        raises(AttributeError, getattr, b, 'x')
-
-    def test_array_buffer(self):
-        if self.runappdirect:
-            skip('PyBufferObject not available outside buffer object.c')
-        module = self.import_extension('foo', [
-            ("roundtrip", "METH_O",
-             """
-                 PyBufferObject *buf = (PyBufferObject *)args;
-                 return PyString_FromStringAndSize(buf->b_ptr, buf->b_size);
-             """),
-            ])
-        import array
-        a = array.array('c', 'text')
-        b = buffer(a)
-        assert module.roundtrip(b) == 'text'
-
-
-    def test_issue2752(self):
-        iterations = 10
-        if self.runappdirect:
-            iterations = 2000
-        module = self.import_extension('foo', [
-            ("test_mod", 'METH_VARARGS',
-            """
-                PyObject *obj;
-                Py_buffer bp;
-                if (!PyArg_ParseTuple(args, "O", &obj))
-                    return NULL;
-
-                if (PyObject_GetBuffer(obj, &bp, PyBUF_SIMPLE) == -1)
-                    return NULL;
-                
-                if (((unsigned char*)bp.buf)[0] != '0') {
-                    void * buf = (void*)bp.buf;
-                    unsigned char val[4];
-                    char * s = PyString_AsString(obj);
-                    memcpy(val, bp.buf, 4);
-                    PyBuffer_Release(&bp);
-                    if (PyObject_GetBuffer(obj, &bp, PyBUF_SIMPLE) == -1)
-                        return NULL;
-                    PyErr_Format(PyExc_ValueError,
-                            "mismatch: %p [%x %x %x %x...] now %p [%x %x %x %x...] as str '%s'",
-                            buf, val[0], val[1], val[2], val[3],
-                            (void *)bp.buf,
-                            ((unsigned char*)bp.buf)[0],
-                            ((unsigned char*)bp.buf)[1],
-                            ((unsigned char*)bp.buf)[2],
-                            ((unsigned char*)bp.buf)[3],
-                            s);
-                    PyBuffer_Release(&bp);
-                    return NULL;
-                }
-
-                PyBuffer_Release(&bp);
-                Py_RETURN_NONE;
-            """),
-            ])
-        bufsize = 4096
-        def getdata(bufsize):
-            data = b'01234567'
-            for x in range(18):
-                data += data
-                if len(data) >= bufsize:
-                    break
-            return data
-        for j in range(iterations):
-            block = getdata(bufsize)
-            assert block[:8] == '01234567'
-            try:
-                module.test_mod(block)
-            except ValueError as e:
-                print("%s at it=%d" % (e, j))
-                assert False
diff --git a/pypy/module/cpyext/test0/test_intobject.py b/pypy/module/cpyext/test0/test_intobject.py
deleted file mode 100644
--- a/pypy/module/cpyext/test0/test_intobject.py
+++ /dev/null
@@ -1,247 +0,0 @@
-from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
-from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from pypy.module.cpyext.intobject import (
-    PyInt_Check, PyInt_AsLong, PyInt_AS_LONG,
-    PyInt_AsUnsignedLong, PyInt_AsUnsignedLongMask,
-    PyInt_AsUnsignedLongLongMask)
-from pypy.module.cpyext.pyobject import (decref, make_ref,
-                                         get_w_obj_and_decref)
-from pypy.module.cpyext.state import State
-import sys
-
-class TestIntObject(BaseApiTest):
-    def test_intobject(self, space):
-        state = space.fromcache(State)
-        assert PyInt_Check(space, space.wrap(3))
-        assert PyInt_Check(space, space.w_True)
-        assert not PyInt_Check(space, space.wrap((1, 2, 3)))
-        for i in [3, -5, -1, -sys.maxint, sys.maxint - 1]:
-            x = PyInt_AsLong(space, space.wrap(i))
-            y = PyInt_AS_LONG(space, space.wrap(i))
-            assert x == i
-            assert y == i
-            py_x = state.C.PyInt_FromLong(x + 1)
-            w_x = get_w_obj_and_decref(space, py_x)
-            assert space.type(w_x) is space.w_int
-            assert space.eq_w(w_x, space.wrap(i + 1))
-
-        with raises_w(space, TypeError):
-            PyInt_AsLong(space, space.w_None)
-
-        with raises_w(space, TypeError):
-            PyInt_AsLong(space, None)
-
-        assert PyInt_AsUnsignedLong(space, space.wrap(sys.maxint)) == sys.maxint
-        with raises_w(space, ValueError):
-            PyInt_AsUnsignedLong(space, space.wrap(-5))
-
-        assert (PyInt_AsUnsignedLongMask(space, space.wrap(sys.maxint))
-                == sys.maxint)
-        assert (PyInt_AsUnsignedLongMask(space, space.wrap(10 ** 30))
-                == 10 ** 30 % ((sys.maxint + 1) * 2))
-
-        assert (PyInt_AsUnsignedLongLongMask(space, space.wrap(sys.maxint))
-                == sys.maxint)
-        assert (PyInt_AsUnsignedLongLongMask(space, space.wrap(10 ** 30))
-                == 10 ** 30 % (2 ** 64))
-
-    def test_freelist_direct(self, space):
-        state = space.fromcache(State)
-        p_x = state.C.PyInt_FromLong(12345678)
-        decref(space, p_x)
-        p_y = state.C.PyInt_FromLong(87654321)
-        # check that the address is the same, i.e. that the freelist did its
-        # job
-        assert p_x == p_y
-        decref(space, p_y)
-
-    def test_freelist_make_ref(self, space):
-        w_x = space.newint(12345678)
-        w_y = space.newint(87654321)
-        p_x = make_ref(space, w_x)
-        decref(space, p_x)
-        p_y = make_ref(space, w_y)
-        # check that the address is the same: note that w_x does NOT keep p_x
-        # alive, because in make_ref we have a special case for ints
-        assert p_x == p_y
-        decref(space, p_y)
-
-    def test_freelist_int_subclass(self, space):
-        w_MyInt = space.appexec([], """():
-            class MyInt(int):
-                pass
-            return MyInt""")
-        w_x = space.call_function(w_MyInt, space.newint(12345678))
-        w_y = space.call_function(w_MyInt, space.newint(87654321))
-        p_x = make_ref(space, w_x)
-        decref(space, p_x)
-        p_y = make_ref(space, w_y)
-        # now the address is different because the freelist does not work for
-        # int subclasses
-        assert p_x != p_y
-        decref(space, p_y)
-
-    def test_coerce(self, space):
-        w_obj = space.appexec([], """():
-            class Coerce(object):
-                def __int__(self):
-                    return 42
-            return Coerce()""")
-        assert PyInt_AsLong(space, w_obj) == 42
-
-class AppTestIntObject(AppTestCpythonExtensionBase):
-    def test_fromstring(self):
-        module = self.import_extension('foo', [
-            ("from_string", "METH_NOARGS",
-             """
-                 return PyInt_FromString("1234", NULL, 16);
-             """),
-            ])
-        assert module.from_string() == 0x1234
-        assert type(module.from_string()) is int
-
-    def test_size_t(self):
-        module = self.import_extension('foo', [
-            ("values", "METH_NOARGS",
-             """
-                 return Py_BuildValue("NNNN",
-                     PyInt_FromSize_t(123),
-                     PyInt_FromSize_t((size_t)-1),
-                     PyInt_FromSsize_t(123),
-                     PyInt_FromSsize_t((size_t)-1));
-             """),
-            ])
-        values = module.values()
-        types = [type(x) for x in values]
-        assert types == [int, long, int, int]
-
-    def test_int_subtype(self):
-        module = self.import_extension(
-            'foo', [
-            ("newEnum", "METH_VARARGS",
-             """
-                EnumObject *enumObj;
-                int intval;
-                PyObject *name;
-
-                if (!PyArg_ParseTuple(args, "Oi", &name, &intval))
-                    return NULL;
-
-                enumObj = PyObject_New(EnumObject, &Enum_Type);
-                if (!enumObj) {
-                    return NULL;
-                }
-
-                enumObj->ob_ival = intval;
-                Py_INCREF(name);
-                enumObj->ob_name = name;
-
-                return (PyObject *)enumObj;
-             """),
-            ],
-            prologue="""
-            #include "structmember.h"
-            typedef struct
-            {
-                PyObject_HEAD
-                long ob_ival;
-                PyObject* ob_name;
-            } EnumObject;
-
-            static void
-            enum_dealloc(PyObject *op)
-            {
-                    Py_DECREF(((EnumObject *)op)->ob_name);
-                    Py_TYPE(op)->tp_free(op);
-            }
-
-            static PyMemberDef enum_members[] = {
-                {"name", T_OBJECT, offsetof(EnumObject, ob_name), 0, NULL},
-                {NULL}  /* Sentinel */
-            };
-
-            PyTypeObject Enum_Type = {
-                PyVarObject_HEAD_INIT(NULL, 0)
-                /*tp_name*/             "Enum",
-                /*tp_basicsize*/        sizeof(EnumObject),
-                /*tp_itemsize*/         0,
-                /*tp_dealloc*/          enum_dealloc,
-                /*tp_print*/            0,
-                /*tp_getattr*/          0,
-                /*tp_setattr*/          0,
-                /*tp_compare*/          0,
-                /*tp_repr*/             0,
-                /*tp_as_number*/        0,
-                /*tp_as_sequence*/      0,
-                /*tp_as_mapping*/       0,
-                /*tp_hash*/             0,
-                /*tp_call*/             0,
-                /*tp_str*/              0,
-                /*tp_getattro*/         0,
-                /*tp_setattro*/         0,
-                /*tp_as_buffer*/        0,
-                /*tp_flags*/            Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,
-                /*tp_doc*/              0,
-                /*tp_traverse*/         0,
-                /*tp_clear*/            0,
-                /*tp_richcompare*/      0,
-                /*tp_weaklistoffset*/   0,
-                /*tp_iter*/             0,
-                /*tp_iternext*/         0,
-                /*tp_methods*/          0,
-                /*tp_members*/          enum_members,
-                /*tp_getset*/           0,
-                /*tp_base*/             0, /* set to &PyInt_Type in init function for MSVC */
-                /*tp_dict*/             0,
-                /*tp_descr_get*/        0,
-                /*tp_descr_set*/        0,
-                /*tp_dictoffset*/       0,
-                /*tp_init*/             0,
-                /*tp_alloc*/            0,
-                /*tp_new*/              0
-            };
-            """, more_init = '''
-                Enum_Type.tp_base = &PyInt_Type;
-                if (PyType_Ready(&Enum_Type) < 0) INITERROR;
-            ''')
-
-        a = module.newEnum("ULTIMATE_ANSWER", 42)
-        assert type(a).__name__ == "Enum"
-        assert isinstance(a, int)
-        assert a == int(a) == 42
-        assert a.name == "ULTIMATE_ANSWER"
-
-    def test_int_cast(self):
-        mod = self.import_extension('foo', [
-                #prove it works for ints
-                ("test_int", "METH_NOARGS",
-                """
-                PyObject * obj = PyInt_FromLong(42);
-                PyObject * val;
-                if (!PyInt_Check(obj)) {
-                    Py_DECREF(obj);
-                    PyErr_SetNone(PyExc_ValueError);
-                    return NULL;
-                }
-                val = PyInt_FromLong(((PyIntObject *)obj)->ob_ival);
-                Py_DECREF(obj);
-                return val;
-                """
-                ),
-                ])
-        i = mod.test_int()
-        assert isinstance(i, int)
-        assert i == 42
-
-    def test_int_macros(self):
-        mod = self.import_extension('foo', [
-                ("test_macros", "METH_NOARGS",
-                """
-                PyObject * obj = PyInt_FromLong(42);
-                PyIntObject * i = (PyIntObject*)obj;
-                PyInt_AS_LONG(obj);
-                PyInt_AS_LONG(i);
-                Py_RETURN_NONE;
-                """
-                ),
-                ])
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -85,8 +85,6 @@
 
     w_mod = check_sys_modules_w(space, modulename)
     if w_mod:
-    if parts[-1] == '':
-        del parts[-1]
         return w_mod
     lock = getimportlock(space)
     try:

From pypy.commits at gmail.com  Wed Aug 29 03:49:38 2018
From: pypy.commits at gmail.com (mattip)
Date: Wed, 29 Aug 2018 00:49:38 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix test for py3
Message-ID: <5b865012.1c69fb81.696ab.c75a@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r95039:553bafd3b1ec
Date: 2018-08-29 09:47 +0200
http://bitbucket.org/pypy/pypy/changeset/553bafd3b1ec/

Log:	fix test for py3

diff --git a/pypy/module/cpyext/test0/test_floatobject.py b/pypy/module/cpyext/test0/test_floatobject.py
--- a/pypy/module/cpyext/test0/test_floatobject.py
+++ b/pypy/module/cpyext/test0/test_floatobject.py
@@ -116,18 +116,18 @@
         # floating-point conversion issues (and to avoid having to
         # conditionalize on compiler support for long double)
         for const_name, const_strval in [
-                ('Py_MATH_PIl', "3.1415926535897932384626433832795029L"),
-                ('Py_MATH_PI', "3.14159265358979323846"),
-                ('Py_MATH_El', "2.7182818284590452353602874713526625L"),
-                ('Py_MATH_E', "2.7182818284590452354"),
-                ('Py_MATH_TAU', "6.2831853071795864769252867665590057683943L"),
+                ('Py_MATH_PIl', b"3.1415926535897932384626433832795029L"),
+                ('Py_MATH_PI', b"3.14159265358979323846"),
+                ('Py_MATH_El', b"2.7182818284590452353602874713526625L"),
+                ('Py_MATH_E', b"2.7182818284590452354"),
+                ('Py_MATH_TAU', b"6.2831853071795864769252867665590057683943L"),
             ]:
             module = self.import_extension('foo_%s' % const_name, [
                 ("test", "METH_NOARGS",
                  """
                  #define xstr(s) str(s)
                  #define str(s) #s
-                 return PyString_FromString(xstr(%s));""" % const_name)
+                 return PyBytes_FromString(xstr(%s));""" % const_name)
             ])
             assert module.test() == const_strval
 

From pypy.commits at gmail.com  Wed Aug 29 05:30:10 2018
From: pypy.commits at gmail.com (mattip)
Date: Wed, 29 Aug 2018 02:30:10 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: Merge py3.5 into branch
Message-ID: <5b8667a2.1c69fb81.32caf.9966@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95040:38112d3a526d
Date: 2018-08-29 10:14 +0200
http://bitbucket.org/pypy/pypy/changeset/38112d3a526d/

Log:	Merge py3.5 into branch

diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py
--- a/pypy/module/cpyext/cdatetime.py
+++ b/pypy/module/cpyext/cdatetime.py
@@ -137,8 +137,9 @@
     If it is a datetime.time or datetime.datetime, it may have tzinfo
     '''
     state = space.fromcache(State)
-    # cannot raise here, so just crash
-    assert len(state.datetimeAPI) > 0
+    if len(state.datetimeAPI) ==0:
+        # can happen in subclassing
+        _PyDateTime_Import(space)
     if state.datetimeAPI[0].c_TimeType == py_obj.c_ob_type:
         py_datetime = rffi.cast(PyDateTime_Time, py_obj)
         w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo'))
diff --git a/pypy/module/cpyext/include/pymath.h b/pypy/module/cpyext/include/pymath.h
--- a/pypy/module/cpyext/include/pymath.h
+++ b/pypy/module/cpyext/include/pymath.h
@@ -6,6 +6,74 @@
 functions and constants
 **************************************************************************/
 
+/* High precision definition of pi and e (Euler)
+ * The values are taken from libc6's math.h.
+ */
+#ifndef Py_MATH_PIl
+#define Py_MATH_PIl 3.1415926535897932384626433832795029L
+#endif
+#ifndef Py_MATH_PI
+#define Py_MATH_PI 3.14159265358979323846
+#endif
+
+#ifndef Py_MATH_El
+#define Py_MATH_El 2.7182818284590452353602874713526625L
+#endif
+
+#ifndef Py_MATH_E
+#define Py_MATH_E 2.7182818284590452354
+#endif
+
+/* Tau (2pi) to 40 digits, taken from tauday.com/tau-digits. */
+#ifndef Py_MATH_TAU
+#define Py_MATH_TAU 6.2831853071795864769252867665590057683943L
+#endif
+
+/* Py_IS_NAN(X)
+ * Return 1 if float or double arg is a NaN, else 0.
+ * Caution:
+ *     X is evaluated more than once.
+ *     This may not work on all platforms.  Each platform has *some*
+ *     way to spell this, though -- override in pyconfig.h if you have
+ *     a platform where it doesn't work.
+ */
+#ifndef Py_IS_NAN
+#if __STDC_VERSION__ >= 199901L || __cplusplus >= 201103L
+#define Py_IS_NAN(X) isnan(X)
+#else
+#define Py_IS_NAN(X) ((X) != (X))
+#endif
+#endif
+
+/* Py_IS_INFINITY(X)
+ * Return 1 if float or double arg is an infinity, else 0.
+ * Caution:
+ *    X is evaluated more than once.
+ *    This implementation may set the underflow flag if |X| is very small;
+ *    it really can't be implemented correctly (& easily) before C99.
+ *    Override in pyconfig.h if you have a better spelling on your platform.
+ */
+#ifndef Py_IS_INFINITY
+#  if __STDC_VERSION__ >= 199901L || __cplusplus >= 201103L
+#    define Py_IS_INFINITY(X) isinf(X)
+#  else
+#    define Py_IS_INFINITY(X) ((X) && ((X)*0.5 == (X)))
+#  endif
+#endif
+
+/* Py_IS_FINITE(X)
+ * Return 1 if float or double arg is neither infinite nor NAN, else 0.
+ * Some compilers (e.g. VisualStudio) have intrisics for this, so a special
+ * macro for this particular test is useful
+ */
+#ifndef Py_IS_FINITE
+#if __STDC_VERSION__ >= 199901L || __cplusplus >= 201103L
+#define Py_IS_FINITE(X) isfinite(X)
+#else
+#define Py_IS_FINITE(X) (!Py_IS_INFINITY(X) && !Py_IS_NAN(X))
+#endif
+#endif
+
 /* HUGE_VAL is supposed to expand to a positive double infinity.  Python
  * uses Py_HUGE_VAL instead because some platforms are broken in this
  * respect.  We used to embed code in pyport.h to try to worm around that,
diff --git a/pypy/module/cpyext/test/test_floatobject.py b/pypy/module/cpyext/test/test_floatobject.py
deleted file mode 100644
--- a/pypy/module/cpyext/test/test_floatobject.py
+++ /dev/null
@@ -1,112 +0,0 @@
-import pytest
-from pypy.interpreter.error import OperationError
-from pypy.module.cpyext.test.test_api import BaseApiTest
-from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from rpython.rtyper.lltypesystem import rffi
-from pypy.module.cpyext.floatobject import (
-    PyFloat_FromDouble, PyFloat_AsDouble, PyFloat_AS_DOUBLE, PyNumber_Float,
-    _PyFloat_Unpack4, _PyFloat_Unpack8)
-
-class TestFloatObject(BaseApiTest):
-    def test_floatobject(self, space):
-        assert space.unwrap(PyFloat_FromDouble(space, 3.14)) == 3.14
-        assert PyFloat_AsDouble(space, space.wrap(23.45)) == 23.45
-        assert PyFloat_AS_DOUBLE(space, space.wrap(23.45)) == 23.45
-        with pytest.raises(OperationError):
-            PyFloat_AsDouble(space, space.w_None)
-
-    def test_coerce(self, space):
-        assert space.type(PyNumber_Float(space, space.wrap(3))) is space.w_float
-        assert space.type(PyNumber_Float(space, space.wrap("3"))) is space.w_float
-
-        w_obj = space.appexec([], """():
-            class Coerce(object):
-                def __float__(self):
-                    return 42.5
-            return Coerce()""")
-        assert space.eq_w(PyNumber_Float(space, w_obj), space.wrap(42.5))
-
-    def test_unpack(self, space):
-        with rffi.scoped_str2charp("\x9a\x99\x99?") as ptr:
-            assert abs(_PyFloat_Unpack4(space, ptr, 1) - 1.2) < 1e-7
-        with rffi.scoped_str2charp("?\x99\x99\x9a") as ptr:
-            assert abs(_PyFloat_Unpack4(space, ptr, 0) - 1.2) < 1e-7
-        with rffi.scoped_str2charp("\x1f\x85\xebQ\xb8\x1e\t@") as ptr:
-            assert abs(_PyFloat_Unpack8(space, ptr, 1) - 3.14) < 1e-15
-        with rffi.scoped_str2charp("@\t\x1e\xb8Q\xeb\x85\x1f") as ptr:
-            assert abs(_PyFloat_Unpack8(space, ptr, 0) - 3.14) < 1e-15
-
-class AppTestFloatObject(AppTestCpythonExtensionBase):
-    def test_fromstring(self):
-        module = self.import_extension('foo', [
-            ("from_string", "METH_NOARGS",
-             """
-                 PyObject* str = PyUnicode_FromString("1234.56");
-                 PyObject* res = PyFloat_FromString(str);
-                 Py_DECREF(str);
-                 return res;
-             """),
-            ])
-        assert module.from_string() == 1234.56
-        assert type(module.from_string()) is float
-
-class AppTestFloatMacros(AppTestCpythonExtensionBase):
-    def test_return_nan(self):
-        import math
-
-        module = self.import_extension('foo', [
-            ("return_nan", "METH_NOARGS",
-             "Py_RETURN_NAN;"),
-            ])
-        assert math.isnan(module.return_nan())
-
-    def test_return_inf(self):
-        import math
-
-        module = self.import_extension('foo', [
-            ("return_inf", "METH_NOARGS",
-             "Py_RETURN_INF(10);"),
-            ])
-        inf = module.return_inf()
-        assert inf > 0
-        assert math.isinf(inf)
-
-    def test_return_inf_negative(self):
-        import math
-
-        module = self.import_extension('foo', [
-            ("return_neginf", "METH_NOARGS",
-             "Py_RETURN_INF(-10);"),
-            ])
-        neginf = module.return_neginf()
-        assert neginf < 0
-        assert math.isinf(neginf)
-
-    def test_macro_accepts_wrong_pointer_type(self):
-        module = self.import_extension('foo', [
-            ("test_macros", "METH_NOARGS",
-             """
-             PyObject* o = PyFloat_FromDouble(1.0);
-             // no PyFloatObject
-             char* dumb_pointer = (char*)o;
-
-             PyFloat_AS_DOUBLE(o);
-             PyFloat_AS_DOUBLE(dumb_pointer);
-
-             Py_RETURN_NONE;"""),
-            ])
-
-    def test_PyFloat_Check(self):
-        module = self.import_extension('foo', [
-            ("test", "METH_NOARGS",
-             """
-             PyObject* pyobj = PyFloat_FromDouble(1.0);
-             PyFloatObject* pfo = (PyFloatObject*)pyobj;
-             int res = (PyFloat_Check(pyobj) +
-                        PyFloat_CheckExact(pyobj) * 10 +
-                        PyFloat_Check(pfo) * 100 +
-                        PyFloat_CheckExact(pfo) * 1000);
-             Py_DecRef(pyobj);
-             return PyLong_FromLong(res);"""),
-            ])
-        assert module.test() == 1111
diff --git a/pypy/module/cpyext/test0/__init__.py b/pypy/module/cpyext/test0/__init__.py
new file mode 100644
diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test0/conftest.py
copy from pypy/module/cpyext/test/conftest.py
copy to pypy/module/cpyext/test0/conftest.py
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test0/test_arraymodule.py
rename from pypy/module/cpyext/test/test_arraymodule.py
rename to pypy/module/cpyext/test0/test_arraymodule.py
diff --git a/pypy/module/cpyext/test/test_boolobject.py b/pypy/module/cpyext/test0/test_boolobject.py
rename from pypy/module/cpyext/test/test_boolobject.py
rename to pypy/module/cpyext/test0/test_boolobject.py
diff --git a/pypy/module/cpyext/test/test_borrow.py b/pypy/module/cpyext/test0/test_borrow.py
rename from pypy/module/cpyext/test/test_borrow.py
rename to pypy/module/cpyext/test0/test_borrow.py
diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test0/test_bytearrayobject.py
rename from pypy/module/cpyext/test/test_bytearrayobject.py
rename to pypy/module/cpyext/test0/test_bytearrayobject.py
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test0/test_bytesobject.py
rename from pypy/module/cpyext/test/test_bytesobject.py
rename to pypy/module/cpyext/test0/test_bytesobject.py
diff --git a/pypy/module/cpyext/test/test_capsule.py b/pypy/module/cpyext/test0/test_capsule.py
rename from pypy/module/cpyext/test/test_capsule.py
rename to pypy/module/cpyext/test0/test_capsule.py
diff --git a/pypy/module/cpyext/test/test_cell.py b/pypy/module/cpyext/test0/test_cell.py
rename from pypy/module/cpyext/test/test_cell.py
rename to pypy/module/cpyext/test0/test_cell.py
diff --git a/pypy/module/cpyext/test/test_classobject.py b/pypy/module/cpyext/test0/test_classobject.py
rename from pypy/module/cpyext/test/test_classobject.py
rename to pypy/module/cpyext/test0/test_classobject.py
diff --git a/pypy/module/cpyext/test/test_codecs.py b/pypy/module/cpyext/test0/test_codecs.py
rename from pypy/module/cpyext/test/test_codecs.py
rename to pypy/module/cpyext/test0/test_codecs.py
diff --git a/pypy/module/cpyext/test/test_complexobject.py b/pypy/module/cpyext/test0/test_complexobject.py
rename from pypy/module/cpyext/test/test_complexobject.py
rename to pypy/module/cpyext/test0/test_complexobject.py
diff --git a/pypy/module/cpyext/test/test_cparser.py b/pypy/module/cpyext/test0/test_cparser.py
rename from pypy/module/cpyext/test/test_cparser.py
rename to pypy/module/cpyext/test0/test_cparser.py
diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test0/test_datetime.py
rename from pypy/module/cpyext/test/test_datetime.py
rename to pypy/module/cpyext/test0/test_datetime.py
diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test0/test_dictobject.py
rename from pypy/module/cpyext/test/test_dictobject.py
rename to pypy/module/cpyext/test0/test_dictobject.py
diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test0/test_eval.py
rename from pypy/module/cpyext/test/test_eval.py
rename to pypy/module/cpyext/test0/test_eval.py
diff --git a/pypy/module/cpyext/test/test_fileobject.py b/pypy/module/cpyext/test0/test_fileobject.py
rename from pypy/module/cpyext/test/test_fileobject.py
rename to pypy/module/cpyext/test0/test_fileobject.py
diff --git a/pypy/module/cpyext/test0/test_floatobject.py b/pypy/module/cpyext/test0/test_floatobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test0/test_floatobject.py
@@ -0,0 +1,196 @@
+import pytest
+from pypy.interpreter.error import OperationError
+from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+from rpython.rtyper.lltypesystem import rffi
+from pypy.module.cpyext.floatobject import (
+    PyFloat_FromDouble, PyFloat_AsDouble, PyFloat_AS_DOUBLE, PyNumber_Float,
+    _PyFloat_Unpack4, _PyFloat_Unpack8)
+
+class TestFloatObject(BaseApiTest):
+    def test_floatobject(self, space):
+        assert space.unwrap(PyFloat_FromDouble(space, 3.14)) == 3.14
+        assert PyFloat_AsDouble(space, space.wrap(23.45)) == 23.45
+        assert PyFloat_AS_DOUBLE(space, space.wrap(23.45)) == 23.45
+        with pytest.raises(OperationError):
+            PyFloat_AsDouble(space, space.w_None)
+
+    def test_coerce(self, space):
+        assert space.type(PyNumber_Float(space, space.wrap(3))) is space.w_float
+        assert space.type(PyNumber_Float(space, space.wrap("3"))) is space.w_float
+
+        w_obj = space.appexec([], """():
+            class Coerce(object):
+                def __float__(self):
+                    return 42.5
+            return Coerce()""")
+        assert space.eq_w(PyNumber_Float(space, w_obj), space.wrap(42.5))
+
+    def test_unpack(self, space):
+        with rffi.scoped_str2charp("\x9a\x99\x99?") as ptr:
+            assert abs(_PyFloat_Unpack4(space, ptr, 1) - 1.2) < 1e-7
+        with rffi.scoped_str2charp("?\x99\x99\x9a") as ptr:
+            assert abs(_PyFloat_Unpack4(space, ptr, 0) - 1.2) < 1e-7
+        with rffi.scoped_str2charp("\x1f\x85\xebQ\xb8\x1e\t@") as ptr:
+            assert abs(_PyFloat_Unpack8(space, ptr, 1) - 3.14) < 1e-15
+        with rffi.scoped_str2charp("@\t\x1e\xb8Q\xeb\x85\x1f") as ptr:
+            assert abs(_PyFloat_Unpack8(space, ptr, 0) - 3.14) < 1e-15
+
+class AppTestFloatObject(AppTestCpythonExtensionBase):
+    def test_fromstring(self):
+        module = self.import_extension('foo', [
+            ("from_string", "METH_NOARGS",
+             """
+                 PyObject* str = PyUnicode_FromString("1234.56");
+                 PyObject* res = PyFloat_FromString(str);
+                 Py_DECREF(str);
+                 return res;
+             """),
+            ])
+        assert module.from_string() == 1234.56
+        assert type(module.from_string()) is float
+
+class AppTestFloatMacros(AppTestCpythonExtensionBase):
+    def test_return_nan(self):
+        import math
+
+        module = self.import_extension('foo', [
+            ("return_nan", "METH_NOARGS",
+             "Py_RETURN_NAN;"),
+            ])
+        assert math.isnan(module.return_nan())
+
+    def test_return_inf(self):
+        import math
+
+        module = self.import_extension('foo', [
+            ("return_inf", "METH_NOARGS",
+             "Py_RETURN_INF(10);"),
+            ])
+        inf = module.return_inf()
+        assert inf > 0
+        assert math.isinf(inf)
+
+    def test_return_inf_negative(self):
+        import math
+
+        module = self.import_extension('foo', [
+            ("return_neginf", "METH_NOARGS",
+             "Py_RETURN_INF(-10);"),
+            ])
+        neginf = module.return_neginf()
+        assert neginf < 0
+        assert math.isinf(neginf)
+
+    def test_macro_accepts_wrong_pointer_type(self):
+        module = self.import_extension('foo', [
+            ("test_macros", "METH_NOARGS",
+             """
+             PyObject* o = PyFloat_FromDouble(1.0);
+             // no PyFloatObject
+             char* dumb_pointer = (char*)o;
+
+             PyFloat_AS_DOUBLE(o);
+             PyFloat_AS_DOUBLE(dumb_pointer);
+
+             Py_RETURN_NONE;"""),
+            ])
+
+    def test_PyFloat_Check(self):
+        module = self.import_extension('foo', [
+            ("test", "METH_NOARGS",
+             """
+             PyObject* pyobj = PyFloat_FromDouble(1.0);
+             PyFloatObject* pfo = (PyFloatObject*)pyobj;
+             int res = (PyFloat_Check(pyobj) +
+                        PyFloat_CheckExact(pyobj) * 10 +
+                        PyFloat_Check(pfo) * 100 +
+                        PyFloat_CheckExact(pfo) * 1000);
+             Py_DecRef(pyobj);
+             return PyLong_FromLong(res);"""),
+            ])
+        assert module.test() == 1111
+
+    def test_pymath_consts(self):
+        # test preprocessor constants in their string form to avoid
+        # floating-point conversion issues (and to avoid having to
+        # conditionalize on compiler support for long double)
+        for const_name, const_strval in [
+                ('Py_MATH_PIl', b"3.1415926535897932384626433832795029L"),
+                ('Py_MATH_PI', b"3.14159265358979323846"),
+                ('Py_MATH_El', b"2.7182818284590452353602874713526625L"),
+                ('Py_MATH_E', b"2.7182818284590452354"),
+                ('Py_MATH_TAU', b"6.2831853071795864769252867665590057683943L"),
+            ]:
+            module = self.import_extension('foo_%s' % const_name, [
+                ("test", "METH_NOARGS",
+                 """
+                 #define xstr(s) str(s)
+                 #define str(s) #s
+                 return PyBytes_FromString(xstr(%s));""" % const_name)
+            ])
+            assert module.test() == const_strval
+
+    def test_Py_IS_NAN(self):
+        module = self.import_extension('foo', [
+            ("test", "METH_O",
+             """
+                 double d = PyFloat_AsDouble(args);
+                 return PyBool_FromLong(Py_IS_NAN(d));
+             """),
+            ])
+        assert not module.test(0)
+        assert not module.test(1)
+        assert not module.test(-1)
+        assert not module.test(float('inf'))
+        assert module.test(float('nan'))
+
+    def test_Py_IS_INFINITY(self):
+        module = self.import_extension('foo', [
+            ("test", "METH_O",
+             """
+                 double d = PyFloat_AsDouble(args);
+                 return PyBool_FromLong(Py_IS_INFINITY(d));
+             """),
+            ])
+        assert not module.test(0)
+        assert not module.test(1)
+        assert not module.test(-1)
+        assert not module.test(float('nan'))
+        assert module.test(float('inf'))
+        assert module.test(float('-inf'))
+
+    def test_Py_IS_FINITE(self):
+        module = self.import_extension('foo', [
+            ("test", "METH_O",
+             """
+                 double d = PyFloat_AsDouble(args);
+                 return PyBool_FromLong(Py_IS_FINITE(d));
+             """),
+            ])
+        assert module.test(0)
+        assert module.test(1)
+        assert module.test(-1)
+        assert not module.test(float('nan'))
+        assert not module.test(float('inf'))
+        assert not module.test(float('-inf'))
+
+    def test_Py_HUGE_VAL(self):
+        module = self.import_extension('foo', [
+            ("test", "METH_NOARGS",
+             """
+                 return PyFloat_FromDouble(Py_HUGE_VAL);
+             """),
+            ])
+        assert module.test() == float('inf')
+
+    def test_Py_NAN(self):
+        module = self.import_extension('foo', [
+            ("test", "METH_NOARGS",
+             """
+                 return PyFloat_FromDouble(Py_NAN);
+             """),
+            ])
+        import struct
+        float_bits = struct.Struct('d').pack
+        assert float_bits(module.test()) == float_bits(float('nan'))
diff --git a/pypy/module/cpyext/test/test_frameobject.py b/pypy/module/cpyext/test0/test_frameobject.py
rename from pypy/module/cpyext/test/test_frameobject.py
rename to pypy/module/cpyext/test0/test_frameobject.py
diff --git a/pypy/module/cpyext/test/test_funcobject.py b/pypy/module/cpyext/test0/test_funcobject.py
rename from pypy/module/cpyext/test/test_funcobject.py
rename to pypy/module/cpyext/test0/test_funcobject.py
diff --git a/pypy/module/cpyext/test/test_genobject.py b/pypy/module/cpyext/test0/test_genobject.py
rename from pypy/module/cpyext/test/test_genobject.py
rename to pypy/module/cpyext/test0/test_genobject.py
diff --git a/pypy/module/cpyext/test/test_getargs.py b/pypy/module/cpyext/test0/test_getargs.py
rename from pypy/module/cpyext/test/test_getargs.py
rename to pypy/module/cpyext/test0/test_getargs.py
diff --git a/pypy/module/cpyext/test/test_import.py b/pypy/module/cpyext/test0/test_import.py
rename from pypy/module/cpyext/test/test_import.py
rename to pypy/module/cpyext/test0/test_import.py
diff --git a/pypy/module/cpyext/test/test_iterator.py b/pypy/module/cpyext/test0/test_iterator.py
rename from pypy/module/cpyext/test/test_iterator.py
rename to pypy/module/cpyext/test0/test_iterator.py
diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test0/test_listobject.py
rename from pypy/module/cpyext/test/test_listobject.py
rename to pypy/module/cpyext/test0/test_listobject.py
diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test0/test_longobject.py
rename from pypy/module/cpyext/test/test_longobject.py
rename to pypy/module/cpyext/test0/test_longobject.py
diff --git a/pypy/module/cpyext/test/test_mapping.py b/pypy/module/cpyext/test0/test_mapping.py
rename from pypy/module/cpyext/test/test_mapping.py
rename to pypy/module/cpyext/test0/test_mapping.py
diff --git a/pypy/module/cpyext/test/test_marshal.py b/pypy/module/cpyext/test0/test_marshal.py
rename from pypy/module/cpyext/test/test_marshal.py
rename to pypy/module/cpyext/test0/test_marshal.py
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test0/test_memoryobject.py
rename from pypy/module/cpyext/test/test_memoryobject.py
rename to pypy/module/cpyext/test0/test_memoryobject.py
diff --git a/pypy/module/cpyext/test/test_methodobject.py b/pypy/module/cpyext/test0/test_methodobject.py
rename from pypy/module/cpyext/test/test_methodobject.py
rename to pypy/module/cpyext/test0/test_methodobject.py
diff --git a/pypy/module/cpyext/test/test_module.py b/pypy/module/cpyext/test0/test_module.py
rename from pypy/module/cpyext/test/test_module.py
rename to pypy/module/cpyext/test0/test_module.py
diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test0/test_ndarrayobject.py
rename from pypy/module/cpyext/test/test_ndarrayobject.py
rename to pypy/module/cpyext/test0/test_ndarrayobject.py
diff --git a/pypy/module/cpyext/test/test_number.py b/pypy/module/cpyext/test0/test_number.py
rename from pypy/module/cpyext/test/test_number.py
rename to pypy/module/cpyext/test0/test_number.py
diff --git a/pypy/module/cpyext/test1/__init__.py b/pypy/module/cpyext/test1/__init__.py
new file mode 100644
diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test1/conftest.py
copy from pypy/module/cpyext/test/conftest.py
copy to pypy/module/cpyext/test1/conftest.py
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test1/test_object.py
rename from pypy/module/cpyext/test/test_object.py
rename to pypy/module/cpyext/test1/test_object.py
diff --git a/pypy/module/cpyext/test1/test_pycobject.py b/pypy/module/cpyext/test1/test_pycobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test1/test_pycobject.py
@@ -0,0 +1,30 @@
+import py
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+
+class AppTestStringObject(AppTestCpythonExtensionBase):
+    def test_pycobject_import(self):
+        module = self.import_extension('foo', [
+            ("set_ptr", "METH_O",
+             """
+                 PyObject *pointer, *module;
+                 void *ptr = PyLong_AsVoidPtr(args);
+                 if (PyErr_Occurred()) return NULL;
+                 pointer = PyCObject_FromVoidPtr(ptr, NULL);
+                 if (PyErr_Occurred()) return NULL;
+                 module = PyImport_ImportModule("foo");
+                 PyModule_AddObject(module, "_ptr", pointer);
+                 Py_DECREF(module);
+                 if (PyErr_Occurred()) return NULL;
+                 Py_RETURN_NONE;
+             """),
+            ("get_ptr", "METH_NOARGS",
+             """
+                 void *ptr = PyCObject_Import("foo", "_ptr");
+                 if (PyErr_Occurred()) return NULL;
+                 return PyLong_FromVoidPtr(ptr);
+             """)])
+        module.set_ptr(1234)
+        assert "PyCObject object" in str(module._ptr)
+        import gc; gc.collect()
+        assert module.get_ptr() == 1234
+        del module._ptr
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test1/test_pyerrors.py
rename from pypy/module/cpyext/test/test_pyerrors.py
rename to pypy/module/cpyext/test1/test_pyerrors.py
diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test1/test_pyfile.py
rename from pypy/module/cpyext/test/test_pyfile.py
rename to pypy/module/cpyext/test1/test_pyfile.py
diff --git a/pypy/module/cpyext/test/test_pysignals.py b/pypy/module/cpyext/test1/test_pysignals.py
rename from pypy/module/cpyext/test/test_pysignals.py
rename to pypy/module/cpyext/test1/test_pysignals.py
diff --git a/pypy/module/cpyext/test/test_pystate.py b/pypy/module/cpyext/test1/test_pystate.py
rename from pypy/module/cpyext/test/test_pystate.py
rename to pypy/module/cpyext/test1/test_pystate.py
diff --git a/pypy/module/cpyext/test/test_pystrtod.py b/pypy/module/cpyext/test1/test_pystrtod.py
rename from pypy/module/cpyext/test/test_pystrtod.py
rename to pypy/module/cpyext/test1/test_pystrtod.py
diff --git a/pypy/module/cpyext/test/test_sequence.py b/pypy/module/cpyext/test1/test_sequence.py
rename from pypy/module/cpyext/test/test_sequence.py
rename to pypy/module/cpyext/test1/test_sequence.py
diff --git a/pypy/module/cpyext/test/test_setobject.py b/pypy/module/cpyext/test1/test_setobject.py
rename from pypy/module/cpyext/test/test_setobject.py
rename to pypy/module/cpyext/test1/test_setobject.py
diff --git a/pypy/module/cpyext/test/test_sliceobject.py b/pypy/module/cpyext/test1/test_sliceobject.py
rename from pypy/module/cpyext/test/test_sliceobject.py
rename to pypy/module/cpyext/test1/test_sliceobject.py
diff --git a/pypy/module/cpyext/test/test_structseq.py b/pypy/module/cpyext/test1/test_structseq.py
rename from pypy/module/cpyext/test/test_structseq.py
rename to pypy/module/cpyext/test1/test_structseq.py
diff --git a/pypy/module/cpyext/test/test_sysmodule.py b/pypy/module/cpyext/test1/test_sysmodule.py
rename from pypy/module/cpyext/test/test_sysmodule.py
rename to pypy/module/cpyext/test1/test_sysmodule.py
diff --git a/pypy/module/cpyext/test/test_thread.py b/pypy/module/cpyext/test1/test_thread.py
rename from pypy/module/cpyext/test/test_thread.py
rename to pypy/module/cpyext/test1/test_thread.py
diff --git a/pypy/module/cpyext/test/test_traceback.py b/pypy/module/cpyext/test1/test_traceback.py
rename from pypy/module/cpyext/test/test_traceback.py
rename to pypy/module/cpyext/test1/test_traceback.py
diff --git a/pypy/module/cpyext/test/test_translate.py b/pypy/module/cpyext/test1/test_translate.py
rename from pypy/module/cpyext/test/test_translate.py
rename to pypy/module/cpyext/test1/test_translate.py
diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test1/test_tupleobject.py
rename from pypy/module/cpyext/test/test_tupleobject.py
rename to pypy/module/cpyext/test1/test_tupleobject.py
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test1/test_typeobject.py
rename from pypy/module/cpyext/test/test_typeobject.py
rename to pypy/module/cpyext/test1/test_typeobject.py
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test1/test_unicodeobject.py
rename from pypy/module/cpyext/test/test_unicodeobject.py
rename to pypy/module/cpyext/test1/test_unicodeobject.py
diff --git a/pypy/module/cpyext/test/test_userslots.py b/pypy/module/cpyext/test1/test_userslots.py
rename from pypy/module/cpyext/test/test_userslots.py
rename to pypy/module/cpyext/test1/test_userslots.py
diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test1/test_version.py
rename from pypy/module/cpyext/test/test_version.py
rename to pypy/module/cpyext/test1/test_version.py
diff --git a/pypy/module/cpyext/test/test_weakref.py b/pypy/module/cpyext/test1/test_weakref.py
rename from pypy/module/cpyext/test/test_weakref.py
rename to pypy/module/cpyext/test1/test_weakref.py
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -302,6 +302,13 @@
         assert ambig == sys.modules.get('ambig')
         assert hasattr(ambig,'imapackage')
 
+    def test_trailing_dot(self):
+        # bug-for-bug compatibility with CPython
+        import sys
+        __import__('pkg.pkg1.')
+        assert 'pkg.pkg1' in sys.modules
+        assert 'pkg.pkg1.' not in sys.modules
+
     def test_from_a(self):
         import sys
         from a import imamodule
@@ -709,7 +716,6 @@
             exec("from pkg.withoutall import *", d)
             assert "" in d
 
-
     def test_import_star_with_bogus___all__(self):
         for case in ["not-imported-yet", "already-imported"]:
             try:
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -388,7 +388,7 @@
                 not self.flags & NPY.ARRAY_F_CONTIGUOUS):
            raise oefmt(errtype, "ndarray is not Fortran contiguous")
         if ((flags & space.BUF_ANY_CONTIGUOUS) == space.BUF_ANY_CONTIGUOUS and
-                not (self.flags & NPY.ARRAY_F_CONTIGUOUS and
+                not (self.flags & NPY.ARRAY_F_CONTIGUOUS or
                      self.flags & NPY.ARRAY_C_CONTIGUOUS)):
            raise oefmt(errtype, "ndarray is not contiguous")
         if ((flags & space.BUF_STRIDES) != space.BUF_STRIDES and
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -874,11 +874,11 @@
 
         if traits.str is unicode:
             if path and path[-1] not in (u'/', u'\\', u':'):
-                path += u'/'
+                path += u'\\'
             mask = path + u'*.*'
         else:
             if path and path[-1] not in ('/', '\\', ':'):
-                path += '/'
+                path += '\\'
             mask = path + '*.*'
 
         filedata = lltype.malloc(win32traits.WIN32_FIND_DATA, flavor='raw')

From pypy.commits at gmail.com  Thu Aug 30 06:26:27 2018
From: pypy.commits at gmail.com (arigo)
Date: Thu, 30 Aug 2018 03:26:27 -0700 (PDT)
Subject: [pypy-commit] cffi default: Issue #379
Message-ID: <5b87c653.1c69fb81.54b6b.d924@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3151:abb60e05d3bf
Date: 2018-08-30 11:36 +0200
http://bitbucket.org/cffi/cffi/changeset/abb60e05d3bf/

Log:	Issue #379

	Accept ``ffi.new("int[4]", p)`` if p is itself another cdata
	``int[4]``.

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -1228,17 +1228,17 @@
     return (cffi_char32_t)-1;
 }
 
-static int _convert_error(PyObject *init, const char *ct_name,
+static int _convert_error(PyObject *init, CTypeDescrObject *ct,
                           const char *expected)
 {
     if (CData_Check(init)) {
-        const char *ct_name_2 = ((CDataObject *)init)->c_type->ct_name;
-        if (strcmp(ct_name, ct_name_2) != 0)
+        CTypeDescrObject *ct2 = ((CDataObject *)init)->c_type;
+        if (strcmp(ct->ct_name, ct2->ct_name) != 0)
             PyErr_Format(PyExc_TypeError,
                          "initializer for ctype '%s' must be a %s, "
                          "not cdata '%s'",
-                         ct_name, expected, ct_name_2);
-        else {
+                         ct->ct_name, expected, ct2->ct_name);
+        else if (ct != ct2) {
             /* in case we'd give the error message "initializer for
                ctype 'A' must be a pointer to same type, not cdata
                'B'", but with A=B, then give instead a different error
@@ -1247,14 +1247,21 @@
                          "initializer for ctype '%s' appears indeed to be '%s',"
                          " but the types are different (check that you are not"
                          " e.g. mixing up different ffi instances)",
-                         ct_name, ct_name_2);
+                         ct->ct_name, ct2->ct_name);
+        }
+        else
+        {
+            PyErr_Format(PyExc_SystemError,
+                         "initializer for ctype '%s' is correct, but we get "
+                         "an internal mismatch--please report a bug",
+                         ct->ct_name);
         }
     }
     else
         PyErr_Format(PyExc_TypeError,
                      "initializer for ctype '%s' must be a %s, "
                      "not %.200s",
-                     ct_name, expected, Py_TYPE(init)->tp_name);
+                     ct->ct_name, expected, Py_TYPE(init)->tp_name);
     return -1;
 }
 
@@ -1368,6 +1375,15 @@
     return 0;
 }
 
+static Py_ssize_t
+get_array_length(CDataObject *cd)
+{
+    if (cd->c_type->ct_length < 0)
+        return ((CDataObject_own_length *)cd)->length;
+    else
+        return cd->c_type->ct_length;
+}
+
 static int
 convert_array_from_object(char *data, CTypeDescrObject *ct, PyObject *init)
 {
@@ -1453,13 +1469,24 @@
     }
 
  cannot_convert:
-    return _convert_error(init, ct->ct_name, expected);
+    if (CData_Check(init))
+    {
+        CDataObject *cd = (CDataObject *)init;
+        if (cd->c_type == ct)
+        {
+            Py_ssize_t n = get_array_length(cd);
+            memmove(data, cd->c_data, n * ctitem->ct_size);
+            return 0;
+        }
+    }
+    return _convert_error(init, ct, expected);
 }
 
 static int
 convert_struct_from_object(char *data, CTypeDescrObject *ct, PyObject *init,
                            Py_ssize_t *optvarsize)
 {
+    /* does not accept 'init' being already a CData */
     const char *expected;
 
     if (force_lazy_struct(ct) <= 0) {
@@ -1506,7 +1533,7 @@
     }
     expected = optvarsize == NULL ? "list or tuple or dict or struct-cdata"
                                   : "list or tuple or dict";
-    return _convert_error(init, ct->ct_name, expected);
+    return _convert_error(init, ct, expected);
 }
 
 #ifdef __GNUC__
@@ -1682,7 +1709,7 @@
     return _convert_overflow(init, ct->ct_name);
 
  cannot_convert:
-    return _convert_error(init, ct->ct_name, expected);
+    return _convert_error(init, ct, expected);
 }
 
 static int
@@ -1742,15 +1769,6 @@
     return 0;
 }
 
-static Py_ssize_t
-get_array_length(CDataObject *cd)
-{
-    if (cd->c_type->ct_length < 0)
-        return ((CDataObject_own_length *)cd)->length;
-    else
-        return cd->c_type->ct_length;
-}
-
 static int
 get_alignment(CTypeDescrObject *ct)
 {
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -1873,7 +1873,7 @@
 
 def test_newp_copying():
     """Test that we can do newp(<type>, <cdata of the given type>) for most
-    types, with the exception of arrays, like in C.
+    types, including same-type arrays.
     """
     BInt = new_primitive_type("int")
     p = newp(new_pointer_type(BInt), cast(BInt, 42))
@@ -1902,8 +1902,9 @@
     a1 = newp(BArray, [1, 2, 3, 4])
     py.test.raises(TypeError, newp, BArray, a1)
     BArray6 = new_array_type(new_pointer_type(BInt), 6)
-    a1 = newp(BArray6, None)
-    py.test.raises(TypeError, newp, BArray6, a1)
+    a1 = newp(BArray6, [10, 20, 30])
+    a2 = newp(BArray6, a1)
+    assert list(a2) == [10, 20, 30, 0, 0, 0]
     #
     s1 = newp(BStructPtr, [42])
     s2 = newp(BStructPtr, s1[0])
diff --git a/cffi/backend_ctypes.py b/cffi/backend_ctypes.py
--- a/cffi/backend_ctypes.py
+++ b/cffi/backend_ctypes.py
@@ -636,6 +636,10 @@
                 if isinstance(init, bytes):
                     init = [init[i:i+1] for i in range(len(init))]
                 else:
+                    if isinstance(init, CTypesGenericArray):
+                        if (len(init) != len(blob) or
+                            not isinstance(init, CTypesArray)):
+                            raise TypeError("length/type mismatch: %s" % (init,))
                     init = tuple(init)
                 if len(init) > len(blob):
                     raise IndexError("too many initializers")
diff --git a/testing/cffi0/backend_tests.py b/testing/cffi0/backend_tests.py
--- a/testing/cffi0/backend_tests.py
+++ b/testing/cffi0/backend_tests.py
@@ -1972,3 +1972,18 @@
         assert seen[1] == 101
         assert seen[2] == 202
         assert seen[3] == 303
+
+    def test_ffi_array_as_init(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("int[4]", [10, 20, 30, 400])
+        q = ffi.new("int[4]", p)
+        assert list(q) == [10, 20, 30, 400]
+        py.test.raises(TypeError, ffi.new, "int[3]", p)
+        py.test.raises(TypeError, ffi.new, "int[5]", p)
+        py.test.raises(TypeError, ffi.new, "int16_t[4]", p)
+        s = ffi.new("struct {int i[4];}*", {'i': p})
+        assert list(s.i) == [10, 20, 30, 400]
+
+    def test_too_many_initializers(self):
+        ffi = FFI(backend=self.Backend())
+        py.test.raises(IndexError, ffi.new, "int[4]", [10, 20, 30, 40, 50])

From pypy.commits at gmail.com  Thu Aug 30 06:26:29 2018
From: pypy.commits at gmail.com (arigo)
Date: Thu, 30 Aug 2018 03:26:29 -0700 (PDT)
Subject: [pypy-commit] cffi default: document abb60e05d3bf
Message-ID: <5b87c655.1c69fb81.a553b.b10e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3152:caee0949f993
Date: 2018-08-30 11:38 +0200
http://bitbucket.org/cffi/cffi/changeset/caee0949f993/

Log:	document abb60e05d3bf

diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -18,6 +18,9 @@
   again.  This makes them independant on the exact CPython version,
   like they are on other platforms.  **It requires virtualenv 16.0.0.**
 
+* Accept an expression like ``ffi.new("int[4]", p)`` if ``p`` is itself
+  another cdata ``int[4]``.
+
 
 v1.11.5
 =======

From pypy.commits at gmail.com  Thu Aug 30 06:26:31 2018
From: pypy.commits at gmail.com (arigo)
Date: Thu, 30 Aug 2018 03:26:31 -0700 (PDT)
Subject: [pypy-commit] cffi default: Use memcpy here
Message-ID: <5b87c657.1c69fb81.16ea3.5286@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3153:1443130c25e1
Date: 2018-08-30 11:53 +0200
http://bitbucket.org/cffi/cffi/changeset/1443130c25e1/

Log:	Use memcpy here

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -1475,7 +1475,7 @@
         if (cd->c_type == ct)
         {
             Py_ssize_t n = get_array_length(cd);
-            memmove(data, cd->c_data, n * ctitem->ct_size);
+            memcpy(data, cd->c_data, n * ctitem->ct_size);
             return 0;
         }
     }

From pypy.commits at gmail.com  Thu Aug 30 06:27:53 2018
From: pypy.commits at gmail.com (arigo)
Date: Thu, 30 Aug 2018 03:27:53 -0700 (PDT)
Subject: [pypy-commit] pypy default: Test and fix: can get a segfault
 instead of an IndexError "too many initializers"
Message-ID: <5b87c6a9.1c69fb81.b30af.ad0b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r95041:beeebc0b35f3
Date: 2018-08-30 12:17 +0200
http://bitbucket.org/pypy/pypy/changeset/beeebc0b35f3/

Log:	Test and fix: can get a segfault instead of an IndexError "too many
	initializers"

diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -51,9 +51,12 @@
     def unpack_list_of_float_items(self, ptr, length):
         return None
 
-    def pack_list_of_items(self, cdata, w_ob):
+    def pack_list_of_items(self, cdata, w_ob, expected_length):
         return False
 
+    def _within_bounds(self, actual_length, expected_length):
+        return expected_length < 0 or actual_length <= expected_length
+
     def newp(self, w_init, allocator):
         space = self.space
         raise oefmt(space.w_TypeError,
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -292,9 +292,10 @@
             return res
         return None
 
-    def pack_list_of_items(self, cdata, w_ob):
+    def pack_list_of_items(self, cdata, w_ob, expected_length):
         int_list = self.space.listview_int(w_ob)
-        if int_list is not None:
+        if (int_list is not None and
+                self._within_bounds(len(int_list), expected_length)):
             if self.size == rffi.sizeof(rffi.LONG): # fastest path
                 from rpython.rlib.rrawarray import copy_list_to_raw_array
                 cdata = rffi.cast(rffi.LONGP, cdata)
@@ -305,7 +306,8 @@
                 if overflowed != 0:
                     self._overflow(self.space.newint(overflowed))
             return True
-        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob,
+                                                   expected_length)
 
 
 class W_CTypePrimitiveUnsigned(W_CTypePrimitive):
@@ -375,15 +377,17 @@
             return res
         return None
 
-    def pack_list_of_items(self, cdata, w_ob):
+    def pack_list_of_items(self, cdata, w_ob, expected_length):
         int_list = self.space.listview_int(w_ob)
-        if int_list is not None:
+        if (int_list is not None and
+                self._within_bounds(len(int_list), expected_length)):
             overflowed = misc.pack_list_to_raw_array_bounds_unsigned(
                 int_list, cdata, self.size, self.vrangemax)
             if overflowed != 0:
                 self._overflow(self.space.newint(overflowed))
             return True
-        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob,
+                                                   expected_length)
 
 
 class W_CTypePrimitiveBool(W_CTypePrimitiveUnsigned):
@@ -471,9 +475,10 @@
             return res
         return None
 
-    def pack_list_of_items(self, cdata, w_ob):
+    def pack_list_of_items(self, cdata, w_ob, expected_length):
         float_list = self.space.listview_float(w_ob)
-        if float_list is not None:
+        if (float_list is not None and
+                self._within_bounds(len(float_list), expected_length)):
             if self.size == rffi.sizeof(rffi.DOUBLE):   # fastest path
                 from rpython.rlib.rrawarray import copy_list_to_raw_array
                 cdata = rffi.cast(rffi.DOUBLEP, cdata)
@@ -483,7 +488,8 @@
                 misc.pack_float_list_to_raw_array(float_list, cdata,
                                                   rffi.FLOAT, rffi.FLOATP)
                 return True
-        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob,
+                                                   expected_length)
 
     def unpack_ptr(self, w_ctypeptr, ptr, length):
         result = self.unpack_list_of_float_items(ptr, length)
@@ -553,13 +559,15 @@
     # 'list(array-of-longdouble)' returns a list of cdata objects,
     # not a list of floats.
 
-    def pack_list_of_items(self, cdata, w_ob):
+    def pack_list_of_items(self, cdata, w_ob, expected_length):
         float_list = self.space.listview_float(w_ob)
-        if float_list is not None:
+        if (float_list is not None and
+                self._within_bounds(len(float_list), expected_length)):
             misc.pack_float_list_to_raw_array(float_list, cdata,
                                              rffi.LONGDOUBLE, rffi.LONGDOUBLEP)
             return True
-        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob,
+                                                   expected_length)
 
     @jit.dont_look_inside
     def nonzero(self, cdata):
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -56,7 +56,7 @@
 
     def _convert_array_from_listview(self, cdata, lst_w):
         space = self.space
-        if self.length >= 0 and len(lst_w) > self.length:
+        if not self._within_bounds(len(lst_w), self.length):
             raise oefmt(space.w_IndexError,
                         "too many initializers for '%s' (got %d)",
                         self.name, len(lst_w))
@@ -69,8 +69,8 @@
         space = self.space
         if (space.isinstance_w(w_ob, space.w_list) or
             space.isinstance_w(w_ob, space.w_tuple)):
-            if self.ctitem.pack_list_of_items(cdata, w_ob):   # fast path
-                pass
+            if self.ctitem.pack_list_of_items(cdata, w_ob, self.length):
+                pass    # fast path
             else:
                 self._convert_array_from_listview(cdata, space.listview(w_ob))
         elif self.accept_str:
diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -267,3 +267,9 @@
         assert lst == [1.25, -2.5, 3.75]
         if not self.runappdirect:
             assert self.get_count() == 1
+
+    def test_too_many_initializers(self):
+        import _cffi_backend
+        ffi = _cffi_backend.FFI()
+        raises(IndexError, ffi.new, "int[4]", [10, 20, 30, 40, 50])
+        raises(IndexError, ffi.new, "int[4]", tuple(range(999)))

From pypy.commits at gmail.com  Thu Aug 30 06:27:55 2018
From: pypy.commits at gmail.com (arigo)
Date: Thu, 30 Aug 2018 03:27:55 -0700 (PDT)
Subject: [pypy-commit] pypy default: more tests, trying to cover all paths
Message-ID: <5b87c6ab.1c69fb81.ae7b7.920d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r95042:60409e95c501
Date: 2018-08-30 12:23 +0200
http://bitbucket.org/pypy/pypy/changeset/60409e95c501/

Log:	more tests, trying to cover all paths

diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -273,3 +273,11 @@
         ffi = _cffi_backend.FFI()
         raises(IndexError, ffi.new, "int[4]", [10, 20, 30, 40, 50])
         raises(IndexError, ffi.new, "int[4]", tuple(range(999)))
+        raises(IndexError, ffi.new, "unsigned int[4]", [10, 20, 30, 40, 50])
+        raises(IndexError, ffi.new, "float[4]", [10, 20, 30, 40, 50])
+        raises(IndexError, ffi.new, "long double[4]", [10, 20, 30, 40, 50])
+        raises(IndexError, ffi.new, "char[4]", [10, 20, 30, 40, 50])
+        raises(IndexError, ffi.new, "wchar_t[4]", [10, 20, 30, 40, 50])
+        raises(IndexError, ffi.new, "_Bool[4]", [10, 20, 30, 40, 50])
+        raises(IndexError, ffi.new, "int[4][4]", [[3,4,5,6]] * 5)
+        raises(IndexError, ffi.new, "int[4][4]", [[3,4,5,6,7]] * 4)

From pypy.commits at gmail.com  Thu Aug 30 06:27:57 2018
From: pypy.commits at gmail.com (arigo)
Date: Thu, 30 Aug 2018 03:27:57 -0700 (PDT)
Subject: [pypy-commit] pypy default: update to cffi/1443130c25e1
Message-ID: <5b87c6ad.1c69fb81.696ab.b44e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r95043:feb6754915bc
Date: 2018-08-30 12:26 +0200
http://bitbucket.org/pypy/pypy/changeset/feb6754915bc/

Log:	update to cffi/1443130c25e1

diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -8,20 +8,43 @@
    the same works for the other two macros.  Py_DEBUG implies them,
    but not the other way around.
 
-   Issue #350 is still open: on Windows, the code here causes it to link
-   with PYTHON36.DLL (for example) instead of PYTHON3.DLL.  A fix was
-   attempted in 164e526a5515 and 14ce6985e1c3, but reverted: virtualenv
-   does not make PYTHON3.DLL available, and so the "correctly" compiled
-   version would not run inside a virtualenv.  We will re-apply the fix
-   after virtualenv has been fixed for some time.  For explanation, see
-   issue #355.  For a workaround if you want PYTHON3.DLL and don't worry
-   about virtualenv, see issue #350.  See also 'py_limited_api' in
-   setuptools_ext.py.
+   The implementation is messy (issue #350): on Windows, with _MSC_VER,
+   we have to define Py_LIMITED_API even before including pyconfig.h.
+   In that case, we guess what pyconfig.h will do to the macros above,
+   and check our guess after the #include.
+
+   Note that on Windows, with CPython 3.x, you need virtualenv version
+   >= 16.0.0.  Older versions don't copy PYTHON3.DLL.  As a workaround
+   you can remove the definition of Py_LIMITED_API here.
+
+   See also 'py_limited_api' in cffi/setuptools_ext.py.
 */
 #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API)
-#  include <pyconfig.h>
-#  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
-#    define Py_LIMITED_API
+#  ifdef _MSC_VER
+#    if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+#      define Py_LIMITED_API
+#    endif
+#    include <pyconfig.h>
+     /* sanity-check: Py_LIMITED_API will cause crashes if any of these
+        are also defined.  Normally, the Python file PC/pyconfig.h does not
+        cause any of these to be defined, with the exception that _DEBUG
+        causes Py_DEBUG.  Double-check that. */
+#    ifdef Py_LIMITED_API
+#      if defined(Py_DEBUG)
+#        error "pyconfig.h unexpectedly defines Py_DEBUG, but Py_LIMITED_API is set"
+#      endif
+#      if defined(Py_TRACE_REFS)
+#        error "pyconfig.h unexpectedly defines Py_TRACE_REFS, but Py_LIMITED_API is set"
+#      endif
+#      if defined(Py_REF_DEBUG)
+#        error "pyconfig.h unexpectedly defines Py_REF_DEBUG, but Py_LIMITED_API is set"
+#      endif
+#    endif
+#  else
+#    include <pyconfig.h>
+#    if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+#      define Py_LIMITED_API
+#    endif
 #  endif
 #endif
 
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -636,6 +636,10 @@
                 if isinstance(init, bytes):
                     init = [init[i:i+1] for i in range(len(init))]
                 else:
+                    if isinstance(init, CTypesGenericArray):
+                        if (len(init) != len(blob) or
+                            not isinstance(init, CTypesArray)):
+                            raise TypeError("length/type mismatch: %s" % (init,))
                     init = tuple(init)
                 if len(init) > len(blob):
                     raise IndexError("too many initializers")
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -81,13 +81,8 @@
     it doesn't so far, creating troubles.  That's why we check
     for "not hasattr(sys, 'gettotalrefcount')" (the 2.7 compatible equivalent
     of 'd' not in sys.abiflags). (http://bugs.python.org/issue28401)
-
-    On Windows, it's better not to use py_limited_api until issue #355
-    can be resolved (by having virtualenv copy PYTHON3.DLL).  See also
-    the start of _cffi_include.h.
     """
-    if ('py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount')
-            and sys.platform != 'win32'):
+    if 'py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount'):
         import setuptools
         try:
             setuptools_major_version = int(setuptools.__version__.partition('.')[0])
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -105,6 +105,11 @@
                 # ctype 'A' must be a pointer to same type, not cdata
                 # 'B'", but with A=B, then give instead a different error
                 # message to try to clear up the confusion
+                if self is w_got.ctype:
+                    raise oefmt(space.w_SystemError,
+                         "initializer for ctype '%s' is correct, but we get "
+                         "an internal mismatch--please report a bug",
+                         self.name)
                 return oefmt(space.w_TypeError,
                              "initializer for ctype '%s' appears indeed to "
                              "be '%s', but the types are different (check "
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -75,7 +75,8 @@
                 self._convert_array_from_listview(cdata, space.listview(w_ob))
         elif self.accept_str:
             if not space.isinstance_w(w_ob, space.w_bytes):
-                raise self._convert_error("str or list or tuple", w_ob)
+                return self._convert_array_from_cdataobj(cdata, w_ob,
+                        "str or list or tuple")
             s = space.bytes_w(w_ob)
             n = len(s)
             if self.length >= 0 and n > self.length:
@@ -90,7 +91,8 @@
         elif isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar):
             from pypy.module._cffi_backend import wchar_helper
             if not space.isinstance_w(w_ob, space.w_unicode):
-                raise self._convert_error("unicode or list or tuple", w_ob)
+                return self._convert_array_from_cdataobj(cdata, w_ob,
+                        "unicode or list or tuple")
             s = space.unicode_w(w_ob)
             if self.ctitem.size == 2:
                 n = wchar_helper.unicode_size_as_char16(s)
@@ -111,7 +113,19 @@
             else:
                 wchar_helper.unicode_to_char32(s, cdata, n, add_final_zero)
         else:
-            raise self._convert_error("list or tuple", w_ob)
+            return self._convert_array_from_cdataobj(cdata, w_ob,
+                    "list or tuple")
+
+    def _convert_array_from_cdataobj(self, cdata, w_ob, errmsg):
+        if isinstance(w_ob, cdataobj.W_CData) and w_ob.ctype is self:
+            length = w_ob.get_array_length()
+            with w_ob as source:
+                source = rffi.cast(rffi.VOIDP, source)
+                target = rffi.cast(rffi.VOIDP, cdata)
+                size = rffi.cast(rffi.SIZE_T, self.ctitem.size * length)
+                rffi.c_memcpy(target, source, size)
+        else:
+            raise self._convert_error(errmsg, w_ob)
 
     def _must_be_string_of_zero_or_one(self, s):
         for c in s:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1862,7 +1862,7 @@
 
 def test_newp_copying():
     """Test that we can do newp(<type>, <cdata of the given type>) for most
-    types, with the exception of arrays, like in C.
+    types, including same-type arrays.
     """
     BInt = new_primitive_type("int")
     p = newp(new_pointer_type(BInt), cast(BInt, 42))
@@ -1891,8 +1891,9 @@
     a1 = newp(BArray, [1, 2, 3, 4])
     py.test.raises(TypeError, newp, BArray, a1)
     BArray6 = new_array_type(new_pointer_type(BInt), 6)
-    a1 = newp(BArray6, None)
-    py.test.raises(TypeError, newp, BArray6, a1)
+    a1 = newp(BArray6, [10, 20, 30])
+    a2 = newp(BArray6, a1)
+    assert list(a2) == [10, 20, 30, 0, 0, 0]
     #
     s1 = newp(BStructPtr, [42])
     s2 = newp(BStructPtr, s1[0])
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -563,3 +563,13 @@
         assert len(z) == 2
         assert ffi.cast("int *", z)[0] == 0x12345
         assert list(z) == [u'\U00012345', u'\x00']   # maybe a 2-unichars str
+
+    def test_ffi_array_as_init(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        p = ffi.new("int[4]", [10, 20, 30, 400])
+        q = ffi.new("int[4]", p)
+        assert list(q) == [10, 20, 30, 400]
+        raises(TypeError, ffi.new, "int[3]", p)
+        raises(TypeError, ffi.new, "int[5]", p)
+        raises(TypeError, ffi.new, "int16_t[4]", p)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1973,3 +1973,18 @@
         assert seen[1] == 101
         assert seen[2] == 202
         assert seen[3] == 303
+
+    def test_ffi_array_as_init(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("int[4]", [10, 20, 30, 400])
+        q = ffi.new("int[4]", p)
+        assert list(q) == [10, 20, 30, 400]
+        py.test.raises(TypeError, ffi.new, "int[3]", p)
+        py.test.raises(TypeError, ffi.new, "int[5]", p)
+        py.test.raises(TypeError, ffi.new, "int16_t[4]", p)
+        s = ffi.new("struct {int i[4];}*", {'i': p})
+        assert list(s.i) == [10, 20, 30, 400]
+
+    def test_too_many_initializers(self):
+        ffi = FFI(backend=self.Backend())
+        py.test.raises(IndexError, ffi.new, "int[4]", [10, 20, 30, 40, 50])

From pypy.commits at gmail.com  Thu Aug 30 07:38:59 2018
From: pypy.commits at gmail.com (arigo)
Date: Thu, 30 Aug 2018 04:38:59 -0700 (PDT)
Subject: [pypy-commit] cffi default: oops,
 don't do that if 'ct' is a CT_POINTER
Message-ID: <5b87d753.1c69fb81.4ab3d.e586@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3154:960ab54c2eb4
Date: 2018-08-30 13:38 +0200
http://bitbucket.org/cffi/cffi/changeset/960ab54c2eb4/

Log:	oops, don't do that if 'ct' is a CT_POINTER

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -1469,7 +1469,7 @@
     }
 
  cannot_convert:
-    if (CData_Check(init))
+    if ((ct->ct_flags & CT_ARRAY) && CData_Check(init))
     {
         CDataObject *cd = (CDataObject *)init;
         if (cd->c_type == ct)

From pypy.commits at gmail.com  Thu Aug 30 07:53:57 2018
From: pypy.commits at gmail.com (arigo)
Date: Thu, 30 Aug 2018 04:53:57 -0700 (PDT)
Subject: [pypy-commit] pypy default: update to cffi/960ab54c2eb4
Message-ID: <5b87dad5.1c69fb81.a112d.1768@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r95044:6df19fd2b6df
Date: 2018-08-30 13:51 +0200
http://bitbucket.org/pypy/pypy/changeset/6df19fd2b6df/

Log:	update to cffi/960ab54c2eb4

diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -104,7 +104,15 @@
         return self.ctptr
 
     def convert_from_object(self, cdata, w_ob):
-        self.convert_array_from_object(cdata, w_ob)
+        if isinstance(w_ob, cdataobj.W_CData) and w_ob.ctype is self:
+            length = w_ob.get_array_length()
+            with w_ob as source:
+                source = rffi.cast(rffi.VOIDP, source)
+                target = rffi.cast(rffi.VOIDP, cdata)
+                size = rffi.cast(rffi.SIZE_T, self.ctitem.size * length)
+                rffi.c_memcpy(target, source, size)
+        else:
+            self.convert_array_from_object(cdata, w_ob)
 
     def convert_to_object(self, cdata):
         if self.length < 0:
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -75,8 +75,7 @@
                 self._convert_array_from_listview(cdata, space.listview(w_ob))
         elif self.accept_str:
             if not space.isinstance_w(w_ob, space.w_bytes):
-                return self._convert_array_from_cdataobj(cdata, w_ob,
-                        "str or list or tuple")
+                raise self._convert_error("str or list or tuple", w_ob)
             s = space.bytes_w(w_ob)
             n = len(s)
             if self.length >= 0 and n > self.length:
@@ -91,8 +90,7 @@
         elif isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar):
             from pypy.module._cffi_backend import wchar_helper
             if not space.isinstance_w(w_ob, space.w_unicode):
-                return self._convert_array_from_cdataobj(cdata, w_ob,
-                        "unicode or list or tuple")
+                raise self._convert_error("unicode or list or tuple", w_ob)
             s = space.unicode_w(w_ob)
             if self.ctitem.size == 2:
                 n = wchar_helper.unicode_size_as_char16(s)
@@ -113,19 +111,7 @@
             else:
                 wchar_helper.unicode_to_char32(s, cdata, n, add_final_zero)
         else:
-            return self._convert_array_from_cdataobj(cdata, w_ob,
-                    "list or tuple")
-
-    def _convert_array_from_cdataobj(self, cdata, w_ob, errmsg):
-        if isinstance(w_ob, cdataobj.W_CData) and w_ob.ctype is self:
-            length = w_ob.get_array_length()
-            with w_ob as source:
-                source = rffi.cast(rffi.VOIDP, source)
-                target = rffi.cast(rffi.VOIDP, cdata)
-                size = rffi.cast(rffi.SIZE_T, self.ctitem.size * length)
-                rffi.c_memcpy(target, source, size)
-        else:
-            raise self._convert_error(errmsg, w_ob)
+            raise self._convert_error("list or tuple", w_ob)
 
     def _must_be_string_of_zero_or_one(self, s):
         for c in s:

From pypy.commits at gmail.com  Thu Aug 30 10:52:32 2018
From: pypy.commits at gmail.com (mattip)
Date: Thu, 30 Aug 2018 07:52:32 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: unicode -> str, str_w -> text_w
Message-ID: <5b8804b0.1c69fb81.1ce84.df18@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95045:4d6b95166120
Date: 2018-08-29 14:55 +0200
http://bitbucket.org/pypy/pypy/changeset/4d6b95166120/

Log:	unicode -> str, str_w -> text_w

diff --git a/pypy/module/__pypy__/test/test_builders.py b/pypy/module/__pypy__/test/test_builders.py
--- a/pypy/module/__pypy__/test/test_builders.py
+++ b/pypy/module/__pypy__/test/test_builders.py
@@ -9,11 +9,11 @@
         b.append("1")
         s = b.build()
         assert s == "abc1231"
-        assert type(s) is unicode
+        assert type(s) is str
         assert b.build() == s
         b.append("123")
         assert b.build() == s + "123"
-        assert type(b.build()) is unicode
+        assert type(b.build()) is str
 
     def test_preallocate(self):
         from __pypy__.builders import StringBuilder
@@ -22,7 +22,7 @@
         b.append("123")
         s = b.build()
         assert s == "abc123"
-        assert type(s) is unicode
+        assert type(s) is str
 
     def test_append_slice(self):
         from __pypy__.builders import StringBuilder
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -767,16 +767,16 @@
     final = space.is_true(w_final)
     state = space.fromcache(CodecState)
     if byteorder == 0:
-        byteorder = 'native'
+        _byteorder = 'native'
     elif byteorder == -1:
-        byteorder = 'little'
+        _byteorder = 'little'
     else:
-        byteorder = 'big'
-    res, lgt, pos = str_decode_utf_16_helper(
+        _byteorder = 'big'
+    res, lgt, pos, bo = str_decode_utf_16_helper(
         data, errors, final,
-        state.decode_error_handler, byteorder)
+        state.decode_error_handler, _byteorder)
     return space.newtuple([space.newutf8(res, lgt),
-                           space.newint(lgt)])
+                           space.newint(lgt), space.newint(bo)])
 
 @unwrap_spec(data='bufferstr', errors='text_or_none', byteorder=int,
              w_final=WrappedDefault(False))
@@ -786,16 +786,16 @@
     final = space.is_true(w_final)
     state = space.fromcache(CodecState)
     if byteorder == 0:
-        byteorder = 'native'
+        _byteorder = 'native'
     elif byteorder == -1:
-        byteorder = 'little'
+        _byteorder = 'little'
     else:
-        byteorder = 'big'
-    res, lgt, pos = str_decode_utf_32_helper(
+        _byteorder = 'big'
+    res, lgt, pos, bo = str_decode_utf_32_helper(
         data, errors, final,
-        state.decode_error_handler, byteorder)
+        state.decode_error_handler, _byteorder)
     return space.newtuple([space.newutf8(res, lgt),
-                           space.newint(lgt)])
+                           space.newint(lgt), space.newint(bo)])
 
 # ____________________________________________________________
 # Charmap
diff --git a/pypy/module/_cppyy/test/test_zjit.py b/pypy/module/_cppyy/test/test_zjit.py
--- a/pypy/module/_cppyy/test/test_zjit.py
+++ b/pypy/module/_cppyy/test/test_zjit.py
@@ -260,7 +260,7 @@
 
     def getattr(self, w_obj, w_name):
         assert isinstance(w_obj, FakeException)
-        assert self.str_w(w_name) == "__name__"
+        assert self.text_w(w_name) == "__name__"
         return FakeString(w_obj.name)
 
     def findattr(self, w_obj, w_name):
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -221,7 +221,7 @@
     w_obj = addr_as_object(rsocket.Address(c_addr, 1 + 2), -1, space)
     assert space.isinstance_w(w_obj, space.w_tuple)
     assert space.int_w(space.getitem(w_obj, space.wrap(0))) == 15
-    assert space.str_w(space.getitem(w_obj, space.wrap(1))) == 'c'
+    assert space.text_w(space.getitem(w_obj, space.wrap(1))) == 'c'
 
 def test_addr_raw_packet():
     from pypy.module._socket.interp_socket import addr_as_object
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -207,7 +207,7 @@
     if w_include_dirs is None:
         return None
     else:
-        return [space.str_w(s) for s in space.listview(w_include_dirs)]
+        return [space.text_w(s) for s in space.listview(w_include_dirs)]
 
 def debug_collect(space):
     rawrefcount._collect()
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -885,7 +885,7 @@
         finally:
             stream.close()
         filename = space.getattr(w_ret, space.wrap('__file__'))
-        assert space.str_w(filename) == u'?'
+        assert space.text_w(filename) == u'?'
 
     def test_parse_source_module(self):
         space = self.space
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -178,8 +178,8 @@
                                                            space.wrap("b")]),
                                             None)
         assert shape == [2]
-        assert space.str_w(elems[0]) == "a"
-        assert space.str_w(elems[1]) == "b"
+        assert space.text_w(elems[0]) == "a"
+        assert space.text_w(elems[1]) == "b"
 
     def test_from_shape_and_storage(self):
         from rpython.rlib.rawstorage import alloc_raw_storage, raw_storage_setitem

From pypy.commits at gmail.com  Thu Aug 30 10:52:36 2018
From: pypy.commits at gmail.com (mattip)
Date: Thu, 30 Aug 2018 07:52:36 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: return bo (byteorder) from
 helpers
Message-ID: <5b8804b4.1c69fb81.b9dc7.9be3@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95046:509a8b3b1a70
Date: 2018-08-29 14:56 +0200
http://bitbucket.org/pypy/pypy/changeset/509a8b3b1a70/

Log:	return bo (byteorder) from helpers

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -1023,15 +1023,15 @@
 
 def str_decode_utf_16(s, errors, final=True,
                       errorhandler=None):
-    return str_decode_utf_16_helper(s, errors, final, errorhandler, "native")
+    return str_decode_utf_16_helper(s, errors, final, errorhandler, "native")[:3]
 
 def str_decode_utf_16_be(s, errors, final=True,
                         errorhandler=None):
-    return str_decode_utf_16_helper(s, errors, final, errorhandler, "big")
+    return str_decode_utf_16_helper(s, errors, final, errorhandler, "big")[:3]
 
 def str_decode_utf_16_le(s, errors, final=True,
                          errorhandler=None):
-    return str_decode_utf_16_helper(s, errors, final, errorhandler, "little")
+    return str_decode_utf_16_helper(s, errors, final, errorhandler, "little")[:3]
 
 def str_decode_utf_16_helper(s, errors, final=True,
                              errorhandler=None,
@@ -1074,7 +1074,7 @@
     else:
         bo = 1
     if size == 0:
-        return '', 0, 0
+        return '', 0, 0, bo
     if bo == -1:
         # force little endian
         ihi = 1
@@ -1134,7 +1134,7 @@
             result.append(r)
     r = result.build()
     lgt = rutf8.check_utf8(r, True)
-    return result.build(), lgt, pos
+    return result.build(), lgt, pos, bo
 
 def _STORECHAR(result, CH, byteorder):
     hi = chr(((CH) >> 8) & 0xff)
@@ -1244,19 +1244,19 @@
                       errorhandler=None):
     return str_decode_utf_32_helper(
         s, errors, final, errorhandler, "native", 'utf-32-' + BYTEORDER2,
-        allow_surrogates=False)
+        allow_surrogates=False)[:3]
 
 def str_decode_utf_32_be(s, errors, final=True,
                          errorhandler=None):
     return str_decode_utf_32_helper(
         s, errors, final, errorhandler, "big", 'utf-32-be',
-        allow_surrogates=False)
+        allow_surrogates=False)[:3]
 
 def str_decode_utf_32_le(s, errors, final=True,
                          errorhandler=None):
     return str_decode_utf_32_helper(
         s, errors, final, errorhandler, "little", 'utf-32-le',
-        allow_surrogates=False)
+        allow_surrogates=False)[:3]
 
 BOM32_DIRECT  = intmask(0x0000FEFF)
 BOM32_REVERSE = intmask(0xFFFE0000)
@@ -1304,7 +1304,7 @@
     else:
         bo = 1
     if size == 0:
-        return '', 0, 0
+        return '', 0, 0, bo
     if bo == -1:
         # force little endian
         iorder = [0, 1, 2, 3]
@@ -1346,7 +1346,7 @@
         pos += 4
     r = result.build()
     lgt = rutf8.check_utf8(r, True)
-    return r, lgt, pos
+    return r, lgt, pos, bo
 
 def _STORECHAR32(result, CH, byteorder):
     c0 = chr(((CH) >> 24) & 0xff)
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -929,11 +929,11 @@
         assert _codecs.utf_16_be_decode(b) == (u'', 0)
         assert _codecs.utf_16_decode(b) == (u'', 0)
         assert _codecs.utf_16_le_decode(b) == (u'', 0)
-        assert _codecs.utf_16_ex_decode(b) == (u'', 0)
+        assert _codecs.utf_16_ex_decode(b) == (u'', 0, 0)
         assert _codecs.utf_32_decode(b) == (u'', 0)
         assert _codecs.utf_32_be_decode(b) == (u'', 0)
         assert _codecs.utf_32_le_decode(b) == (u'', 0)
-        assert _codecs.utf_32_ex_decode(b) == (u'', 0)
+        assert _codecs.utf_32_ex_decode(b) == (u'', 0, 0)
         assert _codecs.charmap_decode(b) == (u'', 0)
         assert _codecs.unicode_escape_decode(b) == (u'', 0)
         assert _codecs.raw_unicode_escape_decode(b) == (u'', 0)
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -20,7 +20,6 @@
 from pypy.module.cpyext.bytesobject import PyBytes_Check, PyBytes_FromObject
 from pypy.module._codecs.interp_codecs import (
     CodecState, latin_1_decode, utf_16_decode, utf_32_decode)
-from pypy.interpreter import unicodehelper
 from pypy.objspace.std import unicodeobject
 import sys
 
@@ -820,10 +819,12 @@
     else:
         errors = 'strict'
 
-    result, length, pos = str_decode_utf_16_helper(
-        string, errors, True, None, byteorder=byteorder)
+    state = space.fromcache(CodecState)
+    result, length, pos, bo = str_decode_utf_16_helper(
+        string, errors, True, state.decode_error_handler,
+        byteorder=byteorder)
     if pbyteorder is not None:
-        pbyteorder[0] = rffi.cast(rffi.INT_real, pos > 0)
+        pbyteorder[0] = rffi.cast(rffi.INT_real, bo)
     return space.newutf8(result, length)
 
 @cpython_api([CONST_STRING, Py_ssize_t, CONST_STRING, INTP_real], PyObject)
@@ -872,10 +873,12 @@
     else:
         errors = 'strict'
 
-    result, length, pos = unicodehelper.str_decode_utf_32_helper(
-        string, errors, True, None, byteorder=byteorder)
+    state = space.fromcache(CodecState)
+    result, length, pos, bo = str_decode_utf_32_helper(
+        string, errors, True, state.decode_error_handler,
+        byteorder=byteorder)
     if pbyteorder is not None:
-        pbyteorder[0] = rffi.cast(rffi.INT_real, pos>0)
+        pbyteorder[0] = rffi.cast(rffi.INT_real, bo)
     return space.newutf8(result, length)
 
 @cpython_api([rffi.CWCHARP, Py_ssize_t, rffi.CCHARP, CONST_STRING],

From pypy.commits at gmail.com  Thu Aug 30 10:52:38 2018
From: pypy.commits at gmail.com (mattip)
Date: Thu, 30 Aug 2018 07:52:38 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: change exception type
Message-ID: <5b8804b6.1c69fb81.48d86.b2c6@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95047:6105ef49d9ca
Date: 2018-08-30 15:14 +0200
http://bitbucket.org/pypy/pypy/changeset/6105ef49d9ca/

Log:	change exception type

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -832,7 +832,7 @@
             # Charmap may return a number
             x = space.int_w(w_ch)
             if not 0 <= x <= 0x10FFFF:
-                raise oefmt(space.w_TypeError,
+                raise oefmt(space.w_ValueError,
                     "character mapping must be in range(0x110000)")
             return rutf8.unichr_as_utf8(x)
         elif space.is_w(w_ch, space.w_None):

From pypy.commits at gmail.com  Fri Aug 31 03:08:30 2018
From: pypy.commits at gmail.com (mattip)
Date: Fri, 31 Aug 2018 00:08:30 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: revert 6105ef49d9ca,
 try to catch error
Message-ID: <5b88e96e.1c69fb81.5add6.ee83@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95048:c07dc57b964d
Date: 2018-08-31 09:04 +0200
http://bitbucket.org/pypy/pypy/changeset/c07dc57b964d/

Log:	revert 6105ef49d9ca, try to catch error

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -832,7 +832,7 @@
             # Charmap may return a number
             x = space.int_w(w_ch)
             if not 0 <= x <= 0x10FFFF:
-                raise oefmt(space.w_ValueError,
+                raise oefmt(space.w_TypeError,
                     "character mapping must be in range(0x110000)")
             return rutf8.unichr_as_utf8(x)
         elif space.is_w(w_ch, space.w_None):
diff --git a/pypy/module/unicodedata/interp_ucd.py b/pypy/module/unicodedata/interp_ucd.py
--- a/pypy/module/unicodedata/interp_ucd.py
+++ b/pypy/module/unicodedata/interp_ucd.py
@@ -289,6 +289,9 @@
     def build(self, space, r, stop):
         builder = Utf8StringBuilder(stop * 3)
         for i in range(stop):
+            code = r_uint(r[i])
+            if code > r_uint(0x10FFFF):
+                raise oefmt(space.w_ValueError, "code > 0x10FFFF")
             builder.append_code(r[i])
         return space.newutf8(builder.build(), stop)
 

From pypy.commits at gmail.com  Fri Aug 31 03:08:32 2018
From: pypy.commits at gmail.com (mattip)
Date: Fri, 31 Aug 2018 00:08:32 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merge default into branch
Message-ID: <5b88e970.1c69fb81.99879.b0e3@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r95049:f7324252bc6b
Date: 2018-08-31 09:05 +0200
http://bitbucket.org/pypy/pypy/changeset/f7324252bc6b/

Log:	merge default into branch

diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -8,20 +8,43 @@
    the same works for the other two macros.  Py_DEBUG implies them,
    but not the other way around.
 
-   Issue #350 is still open: on Windows, the code here causes it to link
-   with PYTHON36.DLL (for example) instead of PYTHON3.DLL.  A fix was
-   attempted in 164e526a5515 and 14ce6985e1c3, but reverted: virtualenv
-   does not make PYTHON3.DLL available, and so the "correctly" compiled
-   version would not run inside a virtualenv.  We will re-apply the fix
-   after virtualenv has been fixed for some time.  For explanation, see
-   issue #355.  For a workaround if you want PYTHON3.DLL and don't worry
-   about virtualenv, see issue #350.  See also 'py_limited_api' in
-   setuptools_ext.py.
+   The implementation is messy (issue #350): on Windows, with _MSC_VER,
+   we have to define Py_LIMITED_API even before including pyconfig.h.
+   In that case, we guess what pyconfig.h will do to the macros above,
+   and check our guess after the #include.
+
+   Note that on Windows, with CPython 3.x, you need virtualenv version
+   >= 16.0.0.  Older versions don't copy PYTHON3.DLL.  As a workaround
+   you can remove the definition of Py_LIMITED_API here.
+
+   See also 'py_limited_api' in cffi/setuptools_ext.py.
 */
 #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API)
-#  include <pyconfig.h>
-#  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
-#    define Py_LIMITED_API
+#  ifdef _MSC_VER
+#    if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+#      define Py_LIMITED_API
+#    endif
+#    include <pyconfig.h>
+     /* sanity-check: Py_LIMITED_API will cause crashes if any of these
+        are also defined.  Normally, the Python file PC/pyconfig.h does not
+        cause any of these to be defined, with the exception that _DEBUG
+        causes Py_DEBUG.  Double-check that. */
+#    ifdef Py_LIMITED_API
+#      if defined(Py_DEBUG)
+#        error "pyconfig.h unexpectedly defines Py_DEBUG, but Py_LIMITED_API is set"
+#      endif
+#      if defined(Py_TRACE_REFS)
+#        error "pyconfig.h unexpectedly defines Py_TRACE_REFS, but Py_LIMITED_API is set"
+#      endif
+#      if defined(Py_REF_DEBUG)
+#        error "pyconfig.h unexpectedly defines Py_REF_DEBUG, but Py_LIMITED_API is set"
+#      endif
+#    endif
+#  else
+#    include <pyconfig.h>
+#    if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+#      define Py_LIMITED_API
+#    endif
 #  endif
 #endif
 
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -636,6 +636,10 @@
                 if isinstance(init, bytes):
                     init = [init[i:i+1] for i in range(len(init))]
                 else:
+                    if isinstance(init, CTypesGenericArray):
+                        if (len(init) != len(blob) or
+                            not isinstance(init, CTypesArray)):
+                            raise TypeError("length/type mismatch: %s" % (init,))
                     init = tuple(init)
                 if len(init) > len(blob):
                     raise IndexError("too many initializers")
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -81,13 +81,8 @@
     it doesn't so far, creating troubles.  That's why we check
     for "not hasattr(sys, 'gettotalrefcount')" (the 2.7 compatible equivalent
     of 'd' not in sys.abiflags). (http://bugs.python.org/issue28401)
-
-    On Windows, it's better not to use py_limited_api until issue #355
-    can be resolved (by having virtualenv copy PYTHON3.DLL).  See also
-    the start of _cffi_include.h.
     """
-    if ('py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount')
-            and sys.platform != 'win32'):
+    if 'py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount'):
         import setuptools
         try:
             setuptools_major_version = int(setuptools.__version__.partition('.')[0])
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -104,7 +104,15 @@
         return self.ctptr
 
     def convert_from_object(self, cdata, w_ob):
-        self.convert_array_from_object(cdata, w_ob)
+        if isinstance(w_ob, cdataobj.W_CData) and w_ob.ctype is self:
+            length = w_ob.get_array_length()
+            with w_ob as source:
+                source = rffi.cast(rffi.VOIDP, source)
+                target = rffi.cast(rffi.VOIDP, cdata)
+                size = rffi.cast(rffi.SIZE_T, self.ctitem.size * length)
+                rffi.c_memcpy(target, source, size)
+        else:
+            self.convert_array_from_object(cdata, w_ob)
 
     def convert_to_object(self, cdata):
         if self.length < 0:
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -51,9 +51,12 @@
     def unpack_list_of_float_items(self, ptr, length):
         return None
 
-    def pack_list_of_items(self, cdata, w_ob):
+    def pack_list_of_items(self, cdata, w_ob, expected_length):
         return False
 
+    def _within_bounds(self, actual_length, expected_length):
+        return expected_length < 0 or actual_length <= expected_length
+
     def newp(self, w_init, allocator):
         space = self.space
         raise oefmt(space.w_TypeError,
@@ -102,6 +105,11 @@
                 # ctype 'A' must be a pointer to same type, not cdata
                 # 'B'", but with A=B, then give instead a different error
                 # message to try to clear up the confusion
+                if self is w_got.ctype:
+                    raise oefmt(space.w_SystemError,
+                         "initializer for ctype '%s' is correct, but we get "
+                         "an internal mismatch--please report a bug",
+                         self.name)
                 return oefmt(space.w_TypeError,
                              "initializer for ctype '%s' appears indeed to "
                              "be '%s', but the types are different (check "
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -292,9 +292,10 @@
             return res
         return None
 
-    def pack_list_of_items(self, cdata, w_ob):
+    def pack_list_of_items(self, cdata, w_ob, expected_length):
         int_list = self.space.listview_int(w_ob)
-        if int_list is not None:
+        if (int_list is not None and
+                self._within_bounds(len(int_list), expected_length)):
             if self.size == rffi.sizeof(rffi.LONG): # fastest path
                 from rpython.rlib.rrawarray import copy_list_to_raw_array
                 cdata = rffi.cast(rffi.LONGP, cdata)
@@ -305,7 +306,8 @@
                 if overflowed != 0:
                     self._overflow(self.space.newint(overflowed))
             return True
-        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob,
+                                                   expected_length)
 
 
 class W_CTypePrimitiveUnsigned(W_CTypePrimitive):
@@ -375,15 +377,17 @@
             return res
         return None
 
-    def pack_list_of_items(self, cdata, w_ob):
+    def pack_list_of_items(self, cdata, w_ob, expected_length):
         int_list = self.space.listview_int(w_ob)
-        if int_list is not None:
+        if (int_list is not None and
+                self._within_bounds(len(int_list), expected_length)):
             overflowed = misc.pack_list_to_raw_array_bounds_unsigned(
                 int_list, cdata, self.size, self.vrangemax)
             if overflowed != 0:
                 self._overflow(self.space.newint(overflowed))
             return True
-        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob,
+                                                   expected_length)
 
 
 class W_CTypePrimitiveBool(W_CTypePrimitiveUnsigned):
@@ -471,9 +475,10 @@
             return res
         return None
 
-    def pack_list_of_items(self, cdata, w_ob):
+    def pack_list_of_items(self, cdata, w_ob, expected_length):
         float_list = self.space.listview_float(w_ob)
-        if float_list is not None:
+        if (float_list is not None and
+                self._within_bounds(len(float_list), expected_length)):
             if self.size == rffi.sizeof(rffi.DOUBLE):   # fastest path
                 from rpython.rlib.rrawarray import copy_list_to_raw_array
                 cdata = rffi.cast(rffi.DOUBLEP, cdata)
@@ -483,7 +488,8 @@
                 misc.pack_float_list_to_raw_array(float_list, cdata,
                                                   rffi.FLOAT, rffi.FLOATP)
                 return True
-        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob,
+                                                   expected_length)
 
     def unpack_ptr(self, w_ctypeptr, ptr, length):
         result = self.unpack_list_of_float_items(ptr, length)
@@ -553,13 +559,15 @@
     # 'list(array-of-longdouble)' returns a list of cdata objects,
     # not a list of floats.
 
-    def pack_list_of_items(self, cdata, w_ob):
+    def pack_list_of_items(self, cdata, w_ob, expected_length):
         float_list = self.space.listview_float(w_ob)
-        if float_list is not None:
+        if (float_list is not None and
+                self._within_bounds(len(float_list), expected_length)):
             misc.pack_float_list_to_raw_array(float_list, cdata,
                                              rffi.LONGDOUBLE, rffi.LONGDOUBLEP)
             return True
-        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob,
+                                                   expected_length)
 
     @jit.dont_look_inside
     def nonzero(self, cdata):
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -58,7 +58,7 @@
 
     def _convert_array_from_listview(self, cdata, lst_w):
         space = self.space
-        if self.length >= 0 and len(lst_w) > self.length:
+        if not self._within_bounds(len(lst_w), self.length):
             raise oefmt(space.w_IndexError,
                         "too many initializers for '%s' (got %d)",
                         self.name, len(lst_w))
@@ -71,8 +71,8 @@
         space = self.space
         if (space.isinstance_w(w_ob, space.w_list) or
             space.isinstance_w(w_ob, space.w_tuple)):
-            if self.ctitem.pack_list_of_items(cdata, w_ob):   # fast path
-                pass
+            if self.ctitem.pack_list_of_items(cdata, w_ob, self.length):
+                pass    # fast path
             else:
                 self._convert_array_from_listview(cdata, space.listview(w_ob))
         elif self.accept_str:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1862,7 +1862,7 @@
 
 def test_newp_copying():
     """Test that we can do newp(<type>, <cdata of the given type>) for most
-    types, with the exception of arrays, like in C.
+    types, including same-type arrays.
     """
     BInt = new_primitive_type("int")
     p = newp(new_pointer_type(BInt), cast(BInt, 42))
@@ -1891,8 +1891,9 @@
     a1 = newp(BArray, [1, 2, 3, 4])
     py.test.raises(TypeError, newp, BArray, a1)
     BArray6 = new_array_type(new_pointer_type(BInt), 6)
-    a1 = newp(BArray6, None)
-    py.test.raises(TypeError, newp, BArray6, a1)
+    a1 = newp(BArray6, [10, 20, 30])
+    a2 = newp(BArray6, a1)
+    assert list(a2) == [10, 20, 30, 0, 0, 0]
     #
     s1 = newp(BStructPtr, [42])
     s2 = newp(BStructPtr, s1[0])
diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -267,3 +267,17 @@
         assert lst == [1.25, -2.5, 3.75]
         if not self.runappdirect:
             assert self.get_count() == 1
+
+    def test_too_many_initializers(self):
+        import _cffi_backend
+        ffi = _cffi_backend.FFI()
+        raises(IndexError, ffi.new, "int[4]", [10, 20, 30, 40, 50])
+        raises(IndexError, ffi.new, "int[4]", tuple(range(999)))
+        raises(IndexError, ffi.new, "unsigned int[4]", [10, 20, 30, 40, 50])
+        raises(IndexError, ffi.new, "float[4]", [10, 20, 30, 40, 50])
+        raises(IndexError, ffi.new, "long double[4]", [10, 20, 30, 40, 50])
+        raises(IndexError, ffi.new, "char[4]", [10, 20, 30, 40, 50])
+        raises(IndexError, ffi.new, "wchar_t[4]", [10, 20, 30, 40, 50])
+        raises(IndexError, ffi.new, "_Bool[4]", [10, 20, 30, 40, 50])
+        raises(IndexError, ffi.new, "int[4][4]", [[3,4,5,6]] * 5)
+        raises(IndexError, ffi.new, "int[4][4]", [[3,4,5,6,7]] * 4)
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -572,3 +572,13 @@
         assert len(z) == 2
         assert ffi.cast("int *", z)[0] == 0x12345
         assert list(z) == [u'\U00012345', u'\x00']   # maybe a 2-unichars str
+
+    def test_ffi_array_as_init(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        p = ffi.new("int[4]", [10, 20, 30, 400])
+        q = ffi.new("int[4]", p)
+        assert list(q) == [10, 20, 30, 400]
+        raises(TypeError, ffi.new, "int[3]", p)
+        raises(TypeError, ffi.new, "int[5]", p)
+        raises(TypeError, ffi.new, "int16_t[4]", p)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1973,3 +1973,18 @@
         assert seen[1] == 101
         assert seen[2] == 202
         assert seen[3] == 303
+
+    def test_ffi_array_as_init(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("int[4]", [10, 20, 30, 400])
+        q = ffi.new("int[4]", p)
+        assert list(q) == [10, 20, 30, 400]
+        py.test.raises(TypeError, ffi.new, "int[3]", p)
+        py.test.raises(TypeError, ffi.new, "int[5]", p)
+        py.test.raises(TypeError, ffi.new, "int16_t[4]", p)
+        s = ffi.new("struct {int i[4];}*", {'i': p})
+        assert list(s.i) == [10, 20, 30, 400]
+
+    def test_too_many_initializers(self):
+        ffi = FFI(backend=self.Backend())
+        py.test.raises(IndexError, ffi.new, "int[4]", [10, 20, 30, 40, 50])

From pypy.commits at gmail.com  Fri Aug 31 03:08:34 2018
From: pypy.commits at gmail.com (mattip)
Date: Fri, 31 Aug 2018 00:08:34 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8-py3: merge py3.5 into branch
Message-ID: <5b88e972.1c69fb81.bf685.5542@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95050:e94950decff4
Date: 2018-08-31 09:06 +0200
http://bitbucket.org/pypy/pypy/changeset/e94950decff4/

Log:	merge py3.5 into branch

diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -8,20 +8,43 @@
    the same works for the other two macros.  Py_DEBUG implies them,
    but not the other way around.
 
-   Issue #350 is still open: on Windows, the code here causes it to link
-   with PYTHON36.DLL (for example) instead of PYTHON3.DLL.  A fix was
-   attempted in 164e526a5515 and 14ce6985e1c3, but reverted: virtualenv
-   does not make PYTHON3.DLL available, and so the "correctly" compiled
-   version would not run inside a virtualenv.  We will re-apply the fix
-   after virtualenv has been fixed for some time.  For explanation, see
-   issue #355.  For a workaround if you want PYTHON3.DLL and don't worry
-   about virtualenv, see issue #350.  See also 'py_limited_api' in
-   setuptools_ext.py.
+   The implementation is messy (issue #350): on Windows, with _MSC_VER,
+   we have to define Py_LIMITED_API even before including pyconfig.h.
+   In that case, we guess what pyconfig.h will do to the macros above,
+   and check our guess after the #include.
+
+   Note that on Windows, with CPython 3.x, you need virtualenv version
+   >= 16.0.0.  Older versions don't copy PYTHON3.DLL.  As a workaround
+   you can remove the definition of Py_LIMITED_API here.
+
+   See also 'py_limited_api' in cffi/setuptools_ext.py.
 */
 #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API)
-#  include <pyconfig.h>
-#  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
-#    define Py_LIMITED_API
+#  ifdef _MSC_VER
+#    if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+#      define Py_LIMITED_API
+#    endif
+#    include <pyconfig.h>
+     /* sanity-check: Py_LIMITED_API will cause crashes if any of these
+        are also defined.  Normally, the Python file PC/pyconfig.h does not
+        cause any of these to be defined, with the exception that _DEBUG
+        causes Py_DEBUG.  Double-check that. */
+#    ifdef Py_LIMITED_API
+#      if defined(Py_DEBUG)
+#        error "pyconfig.h unexpectedly defines Py_DEBUG, but Py_LIMITED_API is set"
+#      endif
+#      if defined(Py_TRACE_REFS)
+#        error "pyconfig.h unexpectedly defines Py_TRACE_REFS, but Py_LIMITED_API is set"
+#      endif
+#      if defined(Py_REF_DEBUG)
+#        error "pyconfig.h unexpectedly defines Py_REF_DEBUG, but Py_LIMITED_API is set"
+#      endif
+#    endif
+#  else
+#    include <pyconfig.h>
+#    if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+#      define Py_LIMITED_API
+#    endif
 #  endif
 #endif
 
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -636,6 +636,10 @@
                 if isinstance(init, bytes):
                     init = [init[i:i+1] for i in range(len(init))]
                 else:
+                    if isinstance(init, CTypesGenericArray):
+                        if (len(init) != len(blob) or
+                            not isinstance(init, CTypesArray)):
+                            raise TypeError("length/type mismatch: %s" % (init,))
                     init = tuple(init)
                 if len(init) > len(blob):
                     raise IndexError("too many initializers")
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -81,13 +81,8 @@
     it doesn't so far, creating troubles.  That's why we check
     for "not hasattr(sys, 'gettotalrefcount')" (the 2.7 compatible equivalent
     of 'd' not in sys.abiflags). (http://bugs.python.org/issue28401)
-
-    On Windows, it's better not to use py_limited_api until issue #355
-    can be resolved (by having virtualenv copy PYTHON3.DLL).  See also
-    the start of _cffi_include.h.
     """
-    if ('py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount')
-            and sys.platform != 'win32'):
+    if 'py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount'):
         import setuptools
         try:
             setuptools_major_version = int(setuptools.__version__.partition('.')[0])
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -104,7 +104,15 @@
         return self.ctptr
 
     def convert_from_object(self, cdata, w_ob):
-        self.convert_array_from_object(cdata, w_ob)
+        if isinstance(w_ob, cdataobj.W_CData) and w_ob.ctype is self:
+            length = w_ob.get_array_length()
+            with w_ob as source:
+                source = rffi.cast(rffi.VOIDP, source)
+                target = rffi.cast(rffi.VOIDP, cdata)
+                size = rffi.cast(rffi.SIZE_T, self.ctitem.size * length)
+                rffi.c_memcpy(target, source, size)
+        else:
+            self.convert_array_from_object(cdata, w_ob)
 
     def convert_to_object(self, cdata):
         if self.length < 0:
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -51,9 +51,12 @@
     def unpack_list_of_float_items(self, ptr, length):
         return None
 
-    def pack_list_of_items(self, cdata, w_ob):
+    def pack_list_of_items(self, cdata, w_ob, expected_length):
         return False
 
+    def _within_bounds(self, actual_length, expected_length):
+        return expected_length < 0 or actual_length <= expected_length
+
     def newp(self, w_init, allocator):
         space = self.space
         raise oefmt(space.w_TypeError,
@@ -102,6 +105,11 @@
                 # ctype 'A' must be a pointer to same type, not cdata
                 # 'B'", but with A=B, then give instead a different error
                 # message to try to clear up the confusion
+                if self is w_got.ctype:
+                    raise oefmt(space.w_SystemError,
+                         "initializer for ctype '%s' is correct, but we get "
+                         "an internal mismatch--please report a bug",
+                         self.name)
                 return oefmt(space.w_TypeError,
                              "initializer for ctype '%s' appears indeed to "
                              "be '%s', but the types are different (check "
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -288,9 +288,10 @@
             return res
         return None
 
-    def pack_list_of_items(self, cdata, w_ob):
+    def pack_list_of_items(self, cdata, w_ob, expected_length):
         int_list = self.space.listview_int(w_ob)
-        if int_list is not None:
+        if (int_list is not None and
+                self._within_bounds(len(int_list), expected_length)):
             if self.size == rffi.sizeof(rffi.LONG): # fastest path
                 from rpython.rlib.rrawarray import copy_list_to_raw_array
                 cdata = rffi.cast(rffi.LONGP, cdata)
@@ -301,7 +302,8 @@
                 if overflowed != 0:
                     self._overflow(self.space.newint(overflowed))
             return True
-        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob,
+                                                   expected_length)
 
 
 class W_CTypePrimitiveUnsigned(W_CTypePrimitive):
@@ -371,15 +373,17 @@
             return res
         return None
 
-    def pack_list_of_items(self, cdata, w_ob):
+    def pack_list_of_items(self, cdata, w_ob, expected_length):
         int_list = self.space.listview_int(w_ob)
-        if int_list is not None:
+        if (int_list is not None and
+                self._within_bounds(len(int_list), expected_length)):
             overflowed = misc.pack_list_to_raw_array_bounds_unsigned(
                 int_list, cdata, self.size, self.vrangemax)
             if overflowed != 0:
                 self._overflow(self.space.newint(overflowed))
             return True
-        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob,
+                                                   expected_length)
 
 
 class W_CTypePrimitiveBool(W_CTypePrimitiveUnsigned):
@@ -467,9 +471,10 @@
             return res
         return None
 
-    def pack_list_of_items(self, cdata, w_ob):
+    def pack_list_of_items(self, cdata, w_ob, expected_length):
         float_list = self.space.listview_float(w_ob)
-        if float_list is not None:
+        if (float_list is not None and
+                self._within_bounds(len(float_list), expected_length)):
             if self.size == rffi.sizeof(rffi.DOUBLE):   # fastest path
                 from rpython.rlib.rrawarray import copy_list_to_raw_array
                 cdata = rffi.cast(rffi.DOUBLEP, cdata)
@@ -479,7 +484,8 @@
                 misc.pack_float_list_to_raw_array(float_list, cdata,
                                                   rffi.FLOAT, rffi.FLOATP)
                 return True
-        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob,
+                                                   expected_length)
 
     def unpack_ptr(self, w_ctypeptr, ptr, length):
         result = self.unpack_list_of_float_items(ptr, length)
@@ -549,13 +555,15 @@
     # 'list(array-of-longdouble)' returns a list of cdata objects,
     # not a list of floats.
 
-    def pack_list_of_items(self, cdata, w_ob):
+    def pack_list_of_items(self, cdata, w_ob, expected_length):
         float_list = self.space.listview_float(w_ob)
-        if float_list is not None:
+        if (float_list is not None and
+                self._within_bounds(len(float_list), expected_length)):
             misc.pack_float_list_to_raw_array(float_list, cdata,
                                              rffi.LONGDOUBLE, rffi.LONGDOUBLEP)
             return True
-        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob,
+                                                   expected_length)
 
     @jit.dont_look_inside
     def nonzero(self, cdata):
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -58,7 +58,7 @@
 
     def _convert_array_from_listview(self, cdata, lst_w):
         space = self.space
-        if self.length >= 0 and len(lst_w) > self.length:
+        if not self._within_bounds(len(lst_w), self.length):
             raise oefmt(space.w_IndexError,
                         "too many initializers for '%s' (got %d)",
                         self.name, len(lst_w))
@@ -71,8 +71,8 @@
         space = self.space
         if (space.isinstance_w(w_ob, space.w_list) or
             space.isinstance_w(w_ob, space.w_tuple)):
-            if self.ctitem.pack_list_of_items(cdata, w_ob):   # fast path
-                pass
+            if self.ctitem.pack_list_of_items(cdata, w_ob, self.length):
+                pass    # fast path
             else:
                 self._convert_array_from_listview(cdata, space.listview(w_ob))
         elif self.accept_str:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1862,7 +1862,7 @@
 
 def test_newp_copying():
     """Test that we can do newp(<type>, <cdata of the given type>) for most
-    types, with the exception of arrays, like in C.
+    types, including same-type arrays.
     """
     BInt = new_primitive_type("int")
     p = newp(new_pointer_type(BInt), cast(BInt, 42))
@@ -1891,8 +1891,9 @@
     a1 = newp(BArray, [1, 2, 3, 4])
     py.test.raises(TypeError, newp, BArray, a1)
     BArray6 = new_array_type(new_pointer_type(BInt), 6)
-    a1 = newp(BArray6, None)
-    py.test.raises(TypeError, newp, BArray6, a1)
+    a1 = newp(BArray6, [10, 20, 30])
+    a2 = newp(BArray6, a1)
+    assert list(a2) == [10, 20, 30, 0, 0, 0]
     #
     s1 = newp(BStructPtr, [42])
     s2 = newp(BStructPtr, s1[0])
diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -267,3 +267,17 @@
         assert lst == [1.25, -2.5, 3.75]
         if not self.runappdirect:
             assert self.get_count() == 1
+
+    def test_too_many_initializers(self):
+        import _cffi_backend
+        ffi = _cffi_backend.FFI()
+        raises(IndexError, ffi.new, "int[4]", [10, 20, 30, 40, 50])
+        raises(IndexError, ffi.new, "int[4]", tuple(range(999)))
+        raises(IndexError, ffi.new, "unsigned int[4]", [10, 20, 30, 40, 50])
+        raises(IndexError, ffi.new, "float[4]", [10, 20, 30, 40, 50])
+        raises(IndexError, ffi.new, "long double[4]", [10, 20, 30, 40, 50])
+        raises(IndexError, ffi.new, "char[4]", [10, 20, 30, 40, 50])
+        raises(IndexError, ffi.new, "wchar_t[4]", [10, 20, 30, 40, 50])
+        raises(IndexError, ffi.new, "_Bool[4]", [10, 20, 30, 40, 50])
+        raises(IndexError, ffi.new, "int[4][4]", [[3,4,5,6]] * 5)
+        raises(IndexError, ffi.new, "int[4][4]", [[3,4,5,6,7]] * 4)
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -572,3 +572,13 @@
         assert len(z) == 2
         assert ffi.cast("int *", z)[0] == 0x12345
         assert list(z) == [u'\U00012345', u'\x00']   # maybe a 2-unichars str
+
+    def test_ffi_array_as_init(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        p = ffi.new("int[4]", [10, 20, 30, 400])
+        q = ffi.new("int[4]", p)
+        assert list(q) == [10, 20, 30, 400]
+        raises(TypeError, ffi.new, "int[3]", p)
+        raises(TypeError, ffi.new, "int[5]", p)
+        raises(TypeError, ffi.new, "int16_t[4]", p)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1973,3 +1973,18 @@
         assert seen[1] == 101
         assert seen[2] == 202
         assert seen[3] == 303
+
+    def test_ffi_array_as_init(self):
+        ffi = FFI(backend=self.Backend())
+        p = ffi.new("int[4]", [10, 20, 30, 400])
+        q = ffi.new("int[4]", p)
+        assert list(q) == [10, 20, 30, 400]
+        py.test.raises(TypeError, ffi.new, "int[3]", p)
+        py.test.raises(TypeError, ffi.new, "int[5]", p)
+        py.test.raises(TypeError, ffi.new, "int16_t[4]", p)
+        s = ffi.new("struct {int i[4];}*", {'i': p})
+        assert list(s.i) == [10, 20, 30, 400]
+
+    def test_too_many_initializers(self):
+        ffi = FFI(backend=self.Backend())
+        py.test.raises(IndexError, ffi.new, "int[4]", [10, 20, 30, 40, 50])

From pypy.commits at gmail.com  Fri Aug 31 03:27:20 2018
From: pypy.commits at gmail.com (arigo)
Date: Fri, 31 Aug 2018 00:27:20 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2879
Message-ID: <5b88edd8.1c69fb81.cef92.a318@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r95051:7756ae58c273
Date: 2018-08-31 09:26 +0200
http://bitbucket.org/pypy/pypy/changeset/7756ae58c273/

Log:	Issue #2879

	another ctypes fix

diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -183,6 +183,7 @@
             self._buffer = self._ffiarray(self._length_, autofree=True)
         for i, arg in enumerate(args):
             self[i] = arg
+    _init_no_arg_ = __init__
 
     def _fix_index(self, index):
         if index < 0:
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -120,7 +120,7 @@
             raise ValueError(
                 "Buffer size too small (%d instead of at least %d bytes)"
                 % (len(buf) + offset, size + offset))
-        result = self()
+        result = self._newowninstance_()
         dest = result._buffer.buffer
         try:
             raw_addr = buf._pypy_raw_address()
@@ -131,6 +131,11 @@
             memmove(dest, raw_addr, size)
         return result
 
+    def _newowninstance_(self):
+        result = self.__new__(self)
+        result._init_no_arg_()
+        return result
+
 
 class CArgObject(object):
     """ simple wrapper around buffer, just for the case of freeing
@@ -162,6 +167,7 @@
 
     def __init__(self, *args, **kwds):
         raise TypeError("%s has no type" % (type(self),))
+    _init_no_arg_ = __init__
 
     def _ensure_objects(self):
         if '_objects' not in self.__dict__:
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -268,6 +268,7 @@
             return
 
         raise TypeError("Unknown constructor %s" % (args,))
+    _init_no_arg_ = __init__
 
     def _wrap_callable(self, to_call, argtypes):
         def f(*args):
@@ -557,7 +558,7 @@
                         keepalive, newarg, newargtype = self._conv_param(argtype, defval)
                     else:
                         import ctypes
-                        val = argtype._type_()
+                        val = argtype._type_._newowninstance_()
                         keepalive = None
                         newarg = ctypes.byref(val)
                         newargtype = type(newarg)
diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -67,8 +67,11 @@
                 self._buffer = ffiarray(1, autofree=True)
             if value is not None:
                 self.contents = value
+        def _init_no_arg_(self):
+            self._buffer = ffiarray(1, autofree=True)
         self._ffiarray = ffiarray
         self.__init__ = __init__
+        self._init_no_arg_ = _init_no_arg_
         self._type_ = TP
 
     def _build_ffiargtype(self):
@@ -137,27 +140,21 @@
     if not (isinstance(tp, _CDataMeta) and tp._is_pointer_like()):
         raise TypeError("cast() argument 2 must be a pointer type, not %s"
                         % (tp,))
+    result = tp._newowninstance_()
     if isinstance(obj, (int, long)):
-        result = tp()
         result._buffer[0] = obj
         return result
     elif obj is None:
-        result = tp()
         return result
     elif isinstance(obj, Array):
-        ptr = tp.__new__(tp)
-        ptr._buffer = tp._ffiarray(1, autofree=True)
-        ptr._buffer[0] = obj._buffer
-        result = ptr
+        result._buffer[0] = obj._buffer
     elif isinstance(obj, bytes):
-        result = tp()
         result._buffer[0] = buffer(obj)._pypy_raw_address()
         return result
     elif not (isinstance(obj, _CData) and type(obj)._is_pointer_like()):
         raise TypeError("cast() argument 1 must be a pointer, not %s"
                         % (type(obj),))
     else:
-        result = tp()
         result._buffer[0] = obj._buffer[0]
 
     # The casted objects '_objects' member:
diff --git a/lib_pypy/_ctypes/primitive.py b/lib_pypy/_ctypes/primitive.py
--- a/lib_pypy/_ctypes/primitive.py
+++ b/lib_pypy/_ctypes/primitive.py
@@ -390,6 +390,7 @@
             self._buffer = self._ffiarray(1, autofree=True)
         if value is not DEFAULT_VALUE:
             self.value = value
+    _init_no_arg_ = __init__
 
     def _ensure_objects(self):
         if self._type_ not in 'zZP':
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -281,6 +281,7 @@
             self.__setattr__(name, arg)
         for name, arg in kwds.items():
             self.__setattr__(name, arg)
+    _init_no_arg_ = __init__
 
     def _subarray(self, fieldtype, name):
         """Return a _rawffi array of length 1 whose address is the same as
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_structures.py b/pypy/module/test_lib_pypy/ctypes_tests/test_structures.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_structures.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_structures.py
@@ -503,6 +503,22 @@
                 assert dostruct(Native) == dostruct(Big)
                 assert dostruct(Native) != dostruct(Little)
 
+    def test_from_buffer_copy(self):
+        from array import array
+
+        class S(Structure):
+            _fields_ = [('i', c_int)]
+            def __init__(self, some, unused, arguments):
+                pass
+        a = array('i', [1234567])
+        s1 = S.from_buffer(a)
+        s2 = S.from_buffer_copy(a)
+        assert s1.i == 1234567
+        assert s2.i == 1234567
+        a[0] = -7654321
+        assert s1.i == -7654321
+        assert s2.i == 1234567
+
 
 class TestPointerMember(BaseCTypesTestChecker):
     def test_1(self):

From pypy.commits at gmail.com  Fri Aug 31 04:11:45 2018
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 31 Aug 2018 01:11:45 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: rm py2-specific test
Message-ID: <5b88f841.1c69fb81.34dfe.600b@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r95052:0fa29dbd5314
Date: 2018-08-31 10:10 +0200
http://bitbucket.org/pypy/pypy/changeset/0fa29dbd5314/

Log:	rm py2-specific test

diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -302,13 +302,6 @@
         assert ambig == sys.modules.get('ambig')
         assert hasattr(ambig,'imapackage')
 
-    def test_trailing_dot(self):
-        # bug-for-bug compatibility with CPython
-        import sys
-        __import__('pkg.pkg1.')
-        assert 'pkg.pkg1' in sys.modules
-        assert 'pkg.pkg1.' not in sys.modules
-
     def test_from_a(self):
         import sys
         from a import imamodule

From pypy.commits at gmail.com  Fri Aug 31 09:03:10 2018
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 31 Aug 2018 06:03:10 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: hg merge py3.5
Message-ID: <5b893c8e.1c69fb81.a9cce.6601@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r95053:103458c3eab6
Date: 2018-08-31 15:00 +0200
http://bitbucket.org/pypy/pypy/changeset/103458c3eab6/

Log:	hg merge py3.5

diff too long, truncating to 2000 out of 4177 lines

diff --git a/lib-python/3/types.py b/lib-python/3/types.py
--- a/lib-python/3/types.py
+++ b/lib-python/3/types.py
@@ -46,9 +46,19 @@
     FrameType = type(tb.tb_frame)
     tb = None; del tb
 
-# For Jython, the following two types are identical
+#
+# On CPython, FunctionType.__code__ is a 'getset_descriptor', but
+# FunctionType.__globals__ is a 'member_descriptor', just like app-level
+# slots.  On PyPy, all descriptors of built-in types are
+# 'getset_descriptor', but the app-level slots are 'member_descriptor'
+# as well.  (On Jython the situation might still be different.)
+#
+# Note that MemberDescriptorType was equal to GetSetDescriptorType in
+# PyPy <= 6.0.
+#
 GetSetDescriptorType = type(FunctionType.__code__)
-MemberDescriptorType = type(FunctionType.__globals__)
+class _C: __slots__ = 's'
+MemberDescriptorType = type(_C.s)
 
 del sys, _f, _g, _C, _c,                           # Not for export
 
diff --git a/lib_pypy/cffi/_cffi_errors.h b/lib_pypy/cffi/_cffi_errors.h
--- a/lib_pypy/cffi/_cffi_errors.h
+++ b/lib_pypy/cffi/_cffi_errors.h
@@ -50,7 +50,9 @@
         "import sys\n"
         "class FileLike:\n"
         "  def write(self, x):\n"
-        "    of.write(x)\n"
+        "    try:\n"
+        "      of.write(x)\n"
+        "    except: pass\n"
         "    self.buf += x\n"
         "fl = FileLike()\n"
         "fl.buf = ''\n"
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -8,20 +8,43 @@
    the same works for the other two macros.  Py_DEBUG implies them,
    but not the other way around.
 
-   Issue #350 is still open: on Windows, the code here causes it to link
-   with PYTHON36.DLL (for example) instead of PYTHON3.DLL.  A fix was
-   attempted in 164e526a5515 and 14ce6985e1c3, but reverted: virtualenv
-   does not make PYTHON3.DLL available, and so the "correctly" compiled
-   version would not run inside a virtualenv.  We will re-apply the fix
-   after virtualenv has been fixed for some time.  For explanation, see
-   issue #355.  For a workaround if you want PYTHON3.DLL and don't worry
-   about virtualenv, see issue #350.  See also 'py_limited_api' in
-   setuptools_ext.py.
+   The implementation is messy (issue #350): on Windows, with _MSC_VER,
+   we have to define Py_LIMITED_API even before including pyconfig.h.
+   In that case, we guess what pyconfig.h will do to the macros above,
+   and check our guess after the #include.
+
+   Note that on Windows, with CPython 3.x, you need virtualenv version
+   >= 16.0.0.  Older versions don't copy PYTHON3.DLL.  As a workaround
+   you can remove the definition of Py_LIMITED_API here.
+
+   See also 'py_limited_api' in cffi/setuptools_ext.py.
 */
 #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API)
-#  include <pyconfig.h>
-#  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
-#    define Py_LIMITED_API
+#  ifdef _MSC_VER
+#    if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+#      define Py_LIMITED_API
+#    endif
+#    include <pyconfig.h>
+     /* sanity-check: Py_LIMITED_API will cause crashes if any of these
+        are also defined.  Normally, the Python file PC/pyconfig.h does not
+        cause any of these to be defined, with the exception that _DEBUG
+        causes Py_DEBUG.  Double-check that. */
+#    ifdef Py_LIMITED_API
+#      if defined(Py_DEBUG)
+#        error "pyconfig.h unexpectedly defines Py_DEBUG, but Py_LIMITED_API is set"
+#      endif
+#      if defined(Py_TRACE_REFS)
+#        error "pyconfig.h unexpectedly defines Py_TRACE_REFS, but Py_LIMITED_API is set"
+#      endif
+#      if defined(Py_REF_DEBUG)
+#        error "pyconfig.h unexpectedly defines Py_REF_DEBUG, but Py_LIMITED_API is set"
+#      endif
+#    endif
+#  else
+#    include <pyconfig.h>
+#    if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+#      define Py_LIMITED_API
+#    endif
 #  endif
 #endif
 
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -636,6 +636,10 @@
                 if isinstance(init, bytes):
                     init = [init[i:i+1] for i in range(len(init))]
                 else:
+                    if isinstance(init, CTypesGenericArray):
+                        if (len(init) != len(blob) or
+                            not isinstance(init, CTypesArray)):
+                            raise TypeError("length/type mismatch: %s" % (init,))
                     init = tuple(init)
                 if len(init) > len(blob):
                     raise IndexError("too many initializers")
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -81,13 +81,8 @@
     it doesn't so far, creating troubles.  That's why we check
     for "not hasattr(sys, 'gettotalrefcount')" (the 2.7 compatible equivalent
     of 'd' not in sys.abiflags). (http://bugs.python.org/issue28401)
-
-    On Windows, it's better not to use py_limited_api until issue #355
-    can be resolved (by having virtualenv copy PYTHON3.DLL).  See also
-    the start of _cffi_include.h.
     """
-    if ('py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount')
-            and sys.platform != 'win32'):
+    if 'py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount'):
         import setuptools
         try:
             setuptools_major_version = int(setuptools.__version__.partition('.')[0])
diff --git a/lib_pypy/grp.py b/lib_pypy/grp.py
--- a/lib_pypy/grp.py
+++ b/lib_pypy/grp.py
@@ -5,8 +5,8 @@
 import os
 from _pwdgrp_cffi import ffi, lib
 import _structseq
-import thread
-_lock = thread.allocate_lock()
+import _thread
+_lock = _thread.allocate_lock()
 
 try: from __pypy__ import builtinify
 except ImportError: builtinify = lambda f: f
diff --git a/lib_pypy/pwd.py b/lib_pypy/pwd.py
--- a/lib_pypy/pwd.py
+++ b/lib_pypy/pwd.py
@@ -12,8 +12,8 @@
 
 from _pwdgrp_cffi import ffi, lib
 import _structseq
-import thread
-_lock = thread.allocate_lock()
+import _thread
+_lock = _thread.allocate_lock()
 
 try: from __pypy__ import builtinify
 except ImportError: builtinify = lambda f: f
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -200,6 +200,10 @@
                default=False,
                requires=[("objspace.usemodules.cpyext", False)]),
 
+    BoolOption("disable_entrypoints_in_cffi",
+               "Disable only cffi's embedding mode.",
+               default=False),
+
     BoolOption("fstrings",
                "if you are really convinced that f-strings are a security "
                "issue, you can disable them here",
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -7,7 +7,8 @@
 
 .. branch: cppyy-packaging
 
-Upgrade to backend 1.2.0, improved handling of templated methods and
+Main items: vastly better template resolution and improved performance. In
+detail: upgrade to backend 1.4, improved handling of templated methods and
 functions (in particular automatic deduction of types), improved pythonization
 interface, range of compatibility fixes for Python3, free functions now take
 fast libffi path when possible, moves for strings (incl. from Python str),
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1513,10 +1513,19 @@
         if readonly and flags & self.BUF_WRITABLE == self.BUF_WRITABLE:
             raise oefmt(self.w_BufferError, "Object is not writable.")
 
+    def _try_buffer_w(self, w_obj, flags):
+        if not we_are_translated():
+            if w_obj.buffer_w.im_func != W_Root.buffer_w.im_func:
+                # when 'buffer_w()' is overridden in the subclass of
+                # W_Root, we need to specify __buffer="read" or
+                # __buffer="read-write" in the TypeDef.
+                assert type(w_obj).typedef.buffer is not None
+        return w_obj.buffer_w(self, flags)
+
     def buffer_w(self, w_obj, flags):
         # New buffer interface, returns a buffer based on flags (PyObject_GetBuffer)
         try:
-            return w_obj.buffer_w(self, flags)
+            return self._try_buffer_w(w_obj, flags)
         except BufferInterfaceNotFound:
             raise oefmt(self.w_TypeError,
                         "'%T' does not support the buffer interface", w_obj)
@@ -1524,14 +1533,14 @@
     def readbuf_w(self, w_obj):
         # Old buffer interface, returns a readonly buffer (PyObject_AsReadBuffer)
         try:
-            return w_obj.buffer_w(self, self.BUF_SIMPLE).as_readbuf()
+            return self._try_buffer_w(w_obj, self.BUF_SIMPLE).as_readbuf()
         except BufferInterfaceNotFound:
             self._getarg_error("bytes-like object", w_obj)
 
     def writebuf_w(self, w_obj):
         # Old buffer interface, returns a writeable buffer (PyObject_AsWriteBuffer)
         try:
-            return w_obj.buffer_w(self, self.BUF_WRITABLE).as_writebuf()
+            return self._try_buffer_w(w_obj, self.BUF_WRITABLE).as_writebuf()
         except (BufferInterfaceNotFound, OperationError):
             self._getarg_error("read-write bytes-like object", w_obj)
 
@@ -1565,7 +1574,7 @@
                 # NB. CPython forbids surrogates here
                 return StringBuffer(w_obj.text_w(self))
             try:
-                return w_obj.buffer_w(self, self.BUF_SIMPLE).as_readbuf()
+                return self._try_buffer_w(w_obj, self.BUF_SIMPLE).as_readbuf()
             except BufferInterfaceNotFound:
                 self._getarg_error("bytes or buffer", w_obj)
         elif code == 's#':
@@ -1577,7 +1586,7 @@
             if self.isinstance_w(w_obj, self.w_unicode):  # NB. CPython forbids
                 return w_obj.text_w(self)                 # surrogates here
             try:
-                return w_obj.buffer_w(self, self.BUF_SIMPLE).as_str()
+                return self._try_buffer_w(w_obj, self.BUF_SIMPLE).as_str()
             except BufferInterfaceNotFound:
                 self._getarg_error("bytes or read-only buffer", w_obj)
         elif code == 'w*':
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -2107,7 +2107,7 @@
         else:
             skip_leading_underscores = False
         for name in all:
-            if skip_leading_underscores and name[0]=='_':
+            if skip_leading_underscores and name and name[0] == '_':
                 continue
             into_locals[name] = getattr(module, name)
 ''', filename=__file__)
diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py
--- a/pypy/interpreter/pyparser/pyparse.py
+++ b/pypy/interpreter/pyparser/pyparse.py
@@ -236,7 +236,7 @@
                             next_token_seen is not None and
                             next_token_seen.value != '('):
                         msg = "Missing parentheses in call to '%s'" % (
-                            last_token_seen,)
+                            last_token_seen.value,)
                     else:
                         msg = "invalid syntax"
                     if e.expected_str is not None:
diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py
--- a/pypy/interpreter/test/test_syntax.py
+++ b/pypy/interpreter/test/test_syntax.py
@@ -699,6 +699,15 @@
             pass
         """
 
+    def test_bug_annotation_inside_nested_function(self):
+        """
+        # this used to crash
+        def f1():
+            def f2(*args: int):
+                pass
+        f1()
+        """
+
 class AppTestSyntaxError:
 
     def test_tokenizer_error_location(self):
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -28,6 +28,9 @@
         self.bases = bases
         # Used in cpyext to fill tp_as_buffer slots
         assert __buffer in {None, 'read-write', 'read'}, "Unknown value for __buffer"
+        for base in bases:
+            if __buffer is None:
+                __buffer = base.buffer
         self.buffer = __buffer
         self.heaptype = False
         self.hasdict = '__dict__' in rawdict
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -193,8 +193,12 @@
 
 @specialize.arg(2)
 def min_max(space, args, implementation_of):
-    if not jit.we_are_jitted() or len(args.arguments_w) != 1 and \
-            jit.loop_unrolling_heuristic(args.arguments_w, len(args.arguments_w)):
+    # the 'normal' version includes a JIT merge point, which will make a
+    # new loop (from the interpreter or from another JIT loop).  If we
+    # give exactly two arguments to the call to max(), or a JIT virtual
+    # list of arguments, then we pick the 'unroll' version with no JIT
+    # merge point.
+    if jit.isvirtual(args.arguments_w) or len(args.arguments_w) == 2:
         return min_max_unroll(space, args, implementation_of)
     else:
         return min_max_normal(space, args, implementation_of)
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -71,7 +71,8 @@
     def __init__(self, space, *args):
         MixedModule.__init__(self, space, *args)
         #
-        if not space.config.objspace.disable_entrypoints:
+        if (not space.config.objspace.disable_entrypoints and
+            not space.config.objspace.disable_entrypoints_in_cffi):
             # import 'embedding', which has the side-effect of registering
             # the 'pypy_init_embedded_cffi_module' entry point
             from pypy.module._cffi_backend import embedding
diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -156,7 +156,7 @@
     return MiniBuffer(LLBuffer(ptr, size), w_cdata)
 
 MiniBuffer.typedef = TypeDef(
-    "_cffi_backend.buffer",
+    "_cffi_backend.buffer", None, None, "read-write",
     __new__ = interp2app(MiniBuffer___new__),
     __len__ = interp2app(MiniBuffer.descr_len),
     __getitem__ = interp2app(MiniBuffer.descr_getitem),
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -104,7 +104,15 @@
         return self.ctptr
 
     def convert_from_object(self, cdata, w_ob):
-        self.convert_array_from_object(cdata, w_ob)
+        if isinstance(w_ob, cdataobj.W_CData) and w_ob.ctype is self:
+            length = w_ob.get_array_length()
+            with w_ob as source:
+                source = rffi.cast(rffi.VOIDP, source)
+                target = rffi.cast(rffi.VOIDP, cdata)
+                size = rffi.cast(rffi.SIZE_T, self.ctitem.size * length)
+                rffi.c_memcpy(target, source, size)
+        else:
+            self.convert_array_from_object(cdata, w_ob)
 
     def convert_to_object(self, cdata):
         if self.length < 0:
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -51,9 +51,12 @@
     def unpack_list_of_float_items(self, ptr, length):
         return None
 
-    def pack_list_of_items(self, cdata, w_ob):
+    def pack_list_of_items(self, cdata, w_ob, expected_length):
         return False
 
+    def _within_bounds(self, actual_length, expected_length):
+        return expected_length < 0 or actual_length <= expected_length
+
     def newp(self, w_init, allocator):
         space = self.space
         raise oefmt(space.w_TypeError,
@@ -102,6 +105,11 @@
                 # ctype 'A' must be a pointer to same type, not cdata
                 # 'B'", but with A=B, then give instead a different error
                 # message to try to clear up the confusion
+                if self is w_got.ctype:
+                    raise oefmt(space.w_SystemError,
+                         "initializer for ctype '%s' is correct, but we get "
+                         "an internal mismatch--please report a bug",
+                         self.name)
                 return oefmt(space.w_TypeError,
                              "initializer for ctype '%s' appears indeed to "
                              "be '%s', but the types are different (check "
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -292,9 +292,10 @@
             return res
         return None
 
-    def pack_list_of_items(self, cdata, w_ob):
+    def pack_list_of_items(self, cdata, w_ob, expected_length):
         int_list = self.space.listview_int(w_ob)
-        if int_list is not None:
+        if (int_list is not None and
+                self._within_bounds(len(int_list), expected_length)):
             if self.size == rffi.sizeof(rffi.LONG): # fastest path
                 from rpython.rlib.rrawarray import copy_list_to_raw_array
                 cdata = rffi.cast(rffi.LONGP, cdata)
@@ -305,7 +306,8 @@
                 if overflowed != 0:
                     self._overflow(self.space.newint(overflowed))
             return True
-        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob,
+                                                   expected_length)
 
 
 class W_CTypePrimitiveUnsigned(W_CTypePrimitive):
@@ -375,15 +377,17 @@
             return res
         return None
 
-    def pack_list_of_items(self, cdata, w_ob):
+    def pack_list_of_items(self, cdata, w_ob, expected_length):
         int_list = self.space.listview_int(w_ob)
-        if int_list is not None:
+        if (int_list is not None and
+                self._within_bounds(len(int_list), expected_length)):
             overflowed = misc.pack_list_to_raw_array_bounds_unsigned(
                 int_list, cdata, self.size, self.vrangemax)
             if overflowed != 0:
                 self._overflow(self.space.newint(overflowed))
             return True
-        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob,
+                                                   expected_length)
 
 
 class W_CTypePrimitiveBool(W_CTypePrimitiveUnsigned):
@@ -471,9 +475,10 @@
             return res
         return None
 
-    def pack_list_of_items(self, cdata, w_ob):
+    def pack_list_of_items(self, cdata, w_ob, expected_length):
         float_list = self.space.listview_float(w_ob)
-        if float_list is not None:
+        if (float_list is not None and
+                self._within_bounds(len(float_list), expected_length)):
             if self.size == rffi.sizeof(rffi.DOUBLE):   # fastest path
                 from rpython.rlib.rrawarray import copy_list_to_raw_array
                 cdata = rffi.cast(rffi.DOUBLEP, cdata)
@@ -483,7 +488,8 @@
                 misc.pack_float_list_to_raw_array(float_list, cdata,
                                                   rffi.FLOAT, rffi.FLOATP)
                 return True
-        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob,
+                                                   expected_length)
 
     def unpack_ptr(self, w_ctypeptr, ptr, length):
         result = self.unpack_list_of_float_items(ptr, length)
@@ -553,13 +559,15 @@
     # 'list(array-of-longdouble)' returns a list of cdata objects,
     # not a list of floats.
 
-    def pack_list_of_items(self, cdata, w_ob):
+    def pack_list_of_items(self, cdata, w_ob, expected_length):
         float_list = self.space.listview_float(w_ob)
-        if float_list is not None:
+        if (float_list is not None and
+                self._within_bounds(len(float_list), expected_length)):
             misc.pack_float_list_to_raw_array(float_list, cdata,
                                              rffi.LONGDOUBLE, rffi.LONGDOUBLEP)
             return True
-        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob)
+        return W_CTypePrimitive.pack_list_of_items(self, cdata, w_ob,
+                                                   expected_length)
 
     @jit.dont_look_inside
     def nonzero(self, cdata):
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -58,7 +58,7 @@
 
     def _convert_array_from_listview(self, cdata, lst_w):
         space = self.space
-        if self.length >= 0 and len(lst_w) > self.length:
+        if not self._within_bounds(len(lst_w), self.length):
             raise oefmt(space.w_IndexError,
                         "too many initializers for '%s' (got %d)",
                         self.name, len(lst_w))
@@ -71,8 +71,8 @@
         space = self.space
         if (space.isinstance_w(w_ob, space.w_list) or
             space.isinstance_w(w_ob, space.w_tuple)):
-            if self.ctitem.pack_list_of_items(cdata, w_ob):   # fast path
-                pass
+            if self.ctitem.pack_list_of_items(cdata, w_ob, self.length):
+                pass    # fast path
             else:
                 self._convert_array_from_listview(cdata, space.listview(w_ob))
         elif self.accept_str:
diff --git a/pypy/module/_cffi_backend/errorbox.py b/pypy/module/_cffi_backend/errorbox.py
--- a/pypy/module/_cffi_backend/errorbox.py
+++ b/pypy/module/_cffi_backend/errorbox.py
@@ -69,7 +69,10 @@
                 import sys
                 class FileLike:
                     def write(self, x):
-                        of.write(x)
+                        try:
+                            of.write(x)
+                        except:
+                            pass
                         self.buf += x
                 fl = FileLike()
                 fl.buf = ''
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1862,7 +1862,7 @@
 
 def test_newp_copying():
     """Test that we can do newp(<type>, <cdata of the given type>) for most
-    types, with the exception of arrays, like in C.
+    types, including same-type arrays.
     """
     BInt = new_primitive_type("int")
     p = newp(new_pointer_type(BInt), cast(BInt, 42))
@@ -1891,8 +1891,9 @@
     a1 = newp(BArray, [1, 2, 3, 4])
     py.test.raises(TypeError, newp, BArray, a1)
     BArray6 = new_array_type(new_pointer_type(BInt), 6)
-    a1 = newp(BArray6, None)
-    py.test.raises(TypeError, newp, BArray6, a1)
+    a1 = newp(BArray6, [10, 20, 30])
+    a2 = newp(BArray6, a1)
+    assert list(a2) == [10, 20, 30, 0, 0, 0]
     #
     s1 = newp(BStructPtr, [42])
     s2 = newp(BStructPtr, s1[0])
diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py
--- a/pypy/module/_cffi_backend/test/test_fastpath.py
+++ b/pypy/module/_cffi_backend/test/test_fastpath.py
@@ -267,3 +267,17 @@
         assert lst == [1.25, -2.5, 3.75]
         if not self.runappdirect:
             assert self.get_count() == 1
+
+    def test_too_many_initializers(self):
+        import _cffi_backend
+        ffi = _cffi_backend.FFI()
+        raises(IndexError, ffi.new, "int[4]", [10, 20, 30, 40, 50])
+        raises(IndexError, ffi.new, "int[4]", tuple(range(999)))
+        raises(IndexError, ffi.new, "unsigned int[4]", [10, 20, 30, 40, 50])
+        raises(IndexError, ffi.new, "float[4]", [10, 20, 30, 40, 50])
+        raises(IndexError, ffi.new, "long double[4]", [10, 20, 30, 40, 50])
+        raises(IndexError, ffi.new, "char[4]", [10, 20, 30, 40, 50])
+        raises(IndexError, ffi.new, "wchar_t[4]", [10, 20, 30, 40, 50])
+        raises(IndexError, ffi.new, "_Bool[4]", [10, 20, 30, 40, 50])
+        raises(IndexError, ffi.new, "int[4][4]", [[3,4,5,6]] * 5)
+        raises(IndexError, ffi.new, "int[4][4]", [[3,4,5,6,7]] * 4)
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -572,3 +572,13 @@
         assert len(z) == 2
         assert ffi.cast("int *", z)[0] == 0x12345
         assert list(z) == [u'\U00012345', u'\x00']   # maybe a 2-unichars str
+
+    def test_ffi_array_as_init(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        p = ffi.new("int[4]", [10, 20, 30, 400])
+        q = ffi.new("int[4]", p)
+        assert list(q) == [10, 20, 30, 400]
+        raises(TypeError, ffi.new, "int[3]", p)
+        raises(TypeError, ffi.new, "int[5]", p)
+        raises(TypeError, ffi.new, "int16_t[4]", p)
diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py
--- a/pypy/module/_cppyy/capi/loadable_capi.py
+++ b/pypy/module/_cppyy/capi/loadable_capi.py
@@ -1,13 +1,18 @@
 import os
+
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib import jit, jit_libffi, libffi, rdynload, objectmodel
 from rpython.rlib.rarithmetic import r_singlefloat
 from rpython.tool import leakfinder
 
-from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.error import oefmt
+from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.argument import Arguments
+from pypy.interpreter.gateway import interp2app, interpindirect2app
+from pypy.interpreter.typedef import TypeDef
+from pypy.objspace.std.iterobject import W_AbstractSeqIterObject
 
+from pypy.module._rawffi.array import W_ArrayInstance
 from pypy.module._cffi_backend import ctypefunc, ctypeprim, cdataobj, misc
 from pypy.module._cffi_backend import newtype
 from pypy.module._cppyy import ffitypes
@@ -23,10 +28,11 @@
 
 class _Arg:         # poor man's union
     _immutable_ = True
-    def __init__(self, tc, h = 0, l = -1, s = '', p = rffi.cast(rffi.VOIDP, 0)):
+    def __init__(self, tc, h = 0, l = -1, d = -1., s = '', p = rffi.cast(rffi.VOIDP, 0)):
         self.tc      = tc
         self._handle = h
         self._long   = l
+        self._double = d
         self._string = s
         self._voidp  = p
 
@@ -40,6 +46,11 @@
     def __init__(self, val):
         _Arg.__init__(self, 'l', l = val)
 
+class _ArgD(_Arg):
+    _immutable_ = True
+    def __init__(self, val):
+        _Arg.__init__(self, 'd', d = val)
+
 class _ArgS(_Arg):
     _immutable_ = True
     def __init__(self, val):
@@ -89,6 +100,9 @@
                     assert obj._voidp != rffi.cast(rffi.VOIDP, 0)
                     data = rffi.cast(rffi.VOIDPP, data)
                     data[0] = obj._voidp
+                elif obj.tc == 'd':
+                    assert isinstance(argtype, ctypeprim.W_CTypePrimitiveFloat)
+                    misc.write_raw_float_data(data, rffi.cast(rffi.DOUBLE, obj._double), argtype.size)
                 else:    # only other use is string
                     assert obj.tc == 's'
                     n = len(obj._string)
@@ -182,6 +196,7 @@
             'call_f'       : ([c_method, c_object, c_int, c_voidp],   c_float),
             'call_d'       : ([c_method, c_object, c_int, c_voidp],   c_double),
             'call_ld'      : ([c_method, c_object, c_int, c_voidp],   c_ldouble),
+            'call_nld'     : ([c_method, c_object, c_int, c_voidp],   c_double),
 
             'call_r'       : ([c_method, c_object, c_int, c_voidp],   c_voidp),
             # call_s actually takes an size_t* as last parameter, but this will do
@@ -236,6 +251,8 @@
             'method_prototype'         : ([c_scope, c_method, c_int], c_ccharp),
             'is_const_method'          : ([c_method],                 c_int),
 
+            'get_num_templated_methods': ([c_scope],                  c_int),
+            'get_templated_method_name': ([c_scope, c_index],         c_ccharp),
             'exists_method_template'   : ([c_scope, c_ccharp],        c_int),
             'method_is_template'       : ([c_scope, c_index],         c_int),
             'get_method_template'      : ([c_scope, c_ccharp, c_ccharp],        c_method),
@@ -272,9 +289,11 @@
             'stdstring2charp'          : ([c_object, c_voidp],        c_ccharp),
             'stdstring2stdstring'      : ([c_object],                 c_object),
 
-            'stdvector_valuetype'      : ([c_ccharp],                 c_ccharp),
-            'stdvector_valuesize'      : ([c_ccharp],                 c_size_t),
+            'longdouble2double'        : ([c_voidp],                  c_double),
+            'double2longdouble'        : ([c_double, c_voidp],        c_void),
 
+            'vectorbool_getitem'       : ([c_object, c_int],          c_int),
+            'vectorbool_setitem'       : ([c_object, c_int, c_int],   c_void),
         }
 
         # size/offset are backend-specific but fixed after load
@@ -401,7 +420,9 @@
     return rffi.cast(rffi.DOUBLE, space.float_w(call_capi(space, 'call_d', args)))
 def c_call_ld(space, cppmethod, cppobject, nargs, cargs):
     args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs)]
-    return rffi.cast(rffi.LONGDOUBLE, space.float_w(call_capi(space, 'call_ld', args)))
+    #return rffi.cast(rffi.LONGDOUBLE, space.float_w(call_capi(space, 'call_ld', args)))
+    # call_nld narrows long double to double
+    return rffi.cast(rffi.DOUBLE, space.float_w(call_capi(space, 'call_nld', args)))
 
 def c_call_r(space, cppmethod, cppobject, nargs, cargs):
     args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs)]
@@ -561,16 +582,21 @@
 def c_is_const_method(space, cppmeth):
     return space.bool_w(call_capi(space, 'is_const_method', [_ArgH(cppmeth)]))
 
+def c_get_num_templated_methods(space, cppscope):
+    return space.int_w(call_capi(space, 'method_is_template', [_ArgH(cppscope.handle)]))
+def c_get_templated_method_name(space, cppscope, index):
+    args = [_ArgH(cppscope.handle), _ArgL(index)]
+    return charp2str_free(space, call_capi(space, 'method_is_template', args))
 def c_exists_method_template(space, cppscope, name):
     args = [_ArgH(cppscope.handle), _ArgS(name)]
     return space.bool_w(call_capi(space, 'exists_method_template', args))
 def c_method_is_template(space, cppscope, index):
     args = [_ArgH(cppscope.handle), _ArgL(index)]
     return space.bool_w(call_capi(space, 'method_is_template', args))
-
 def c_get_method_template(space, cppscope, name, proto):
     args = [_ArgH(cppscope.handle), _ArgS(name), _ArgS(proto)]
     return rffi.cast(C_METHOD, space.uint_w(call_capi(space, 'get_method_template', args)))
+
 def c_get_global_operator(space, nss, lc, rc, op):
     if nss is not None:
         args = [_ArgH(nss.handle), _ArgH(lc.handle), _ArgH(rc.handle), _ArgS(op)]
@@ -619,7 +645,7 @@
     return space.bool_w(call_capi(space, 'is_enum_data', args))
 def c_get_dimension_size(space, cppscope, datamember_index, dim_idx):
     args = [_ArgH(cppscope.handle), _ArgL(datamember_index), _ArgL(dim_idx)]
-    return space.bool_w(call_capi(space, 'get_dimension_size', args))
+    return space.int_w(call_capi(space, 'get_dimension_size', args))
 
 # misc helpers ---------------------------------------------------------------
 def c_strtoll(space, svalue):
@@ -650,24 +676,94 @@
 def c_stdstring2stdstring(space, cppobject):
     return _cdata_to_cobject(space, call_capi(space, 'stdstring2stdstring', [_ArgH(cppobject)]))
 
-def c_stdvector_valuetype(space, pystr):
-    return charp2str_free(space, call_capi(space, 'stdvector_valuetype', [_ArgS(pystr)]))
+def c_longdouble2double(space, addr):
+    return space.float_w(call_capi(space, 'longdouble2double', [_ArgP(addr)]))
+def c_double2longdouble(space, dval, addr):
+    call_capi(space, 'double2longdouble', [_ArgD(dval), _ArgP(addr)])
 
-def c_stdvector_valuetype(space, pystr):
-    return charp2str_free(space, call_capi(space, 'stdvector_valuetype', [_ArgS(pystr)]))
-def c_stdvector_valuesize(space, pystr):
-    return _cdata_to_size_t(space, call_capi(space, 'stdvector_valuesize', [_ArgS(pystr)]))
+def c_vectorbool_getitem(space, vbool, idx):
+    return call_capi(space, 'vectorbool_getitem', [_ArgH(vbool), _ArgL(idx)])
+def c_vectorbool_setitem(space, vbool, idx, value):
+    call_capi(space, 'vectorbool_setitem', [_ArgH(vbool), _ArgL(idx), _ArgL(value)])
 
 
 # TODO: factor these out ...
 # pythonizations
 def stdstring_c_str(space, w_self):
     """Return a python string taking into account \0"""
-
     from pypy.module._cppyy import interp_cppyy
     cppstr = space.interp_w(interp_cppyy.W_CPPInstance, w_self, can_be_None=False)
     return space.newtext(c_stdstring2charp(space, cppstr._rawobject))
 
+def vbool_getindex(space, w_vbool, w_idx):
+    idx = space.getindex_w(w_idx, space.w_IndexError, "std::vector<bool> index")
+    sz = space.len_w(w_vbool)
+    if idx < 0: idx += sz
+    if idx < 0 or idx >= sz:
+        raise IndexError
+    return idx
+
+def vectorbool_getitem(space, w_self, w_idx):
+    """Index a std::vector<bool>, return the value"""
+    from pypy.module._cppyy import interp_cppyy
+    vbool = space.interp_w(interp_cppyy.W_CPPInstance, w_self, can_be_None=False)
+    idx = vbool_getindex(space, w_self, w_idx)
+    item = c_vectorbool_getitem(space, vbool._rawobject, idx)
+    return space.newbool(space.is_true(item))
+
+def vectorbool_setitem(space, w_self, w_idx, w_value):
+    """Index a std::vector<bool>, set the value"""
+    from pypy.module._cppyy import interp_cppyy
+    vbool = space.interp_w(interp_cppyy.W_CPPInstance, w_self, can_be_None=False)
+    idx = vbool_getindex(space, w_self, w_idx)
+    c_vectorbool_setitem(space, vbool._rawobject, idx, int(space.is_true(w_value)))
+
+class W_STLVectorIter(W_AbstractSeqIterObject):
+    # w_seq and index are in base class
+    _immutable_fields_ = ['converter', 'data', 'len', 'stride']
+
+    def __init__(self, space, w_vector):
+        W_AbstractSeqIterObject.__init__(self, w_vector)
+        # TODO: this should live in rpythonize.py or something so that the
+        # imports can move to the top w/o getting circles
+        from pypy.module._cppyy import interp_cppyy
+        assert isinstance(w_vector, interp_cppyy.W_CPPInstance)
+        vector = space.interp_w(interp_cppyy.W_CPPInstance, w_vector)
+
+        v_type = c_resolve_name(space, vector.clsdecl.name+'::value_type')
+        v_size = c_size_of_type(space, v_type)
+
+        if not v_type or not v_size:
+            raise NotImplementedError   # fallback on getitem
+
+        from pypy.module._cppyy import converter
+        self.converter = converter.get_converter(space, v_type, '')
+
+        # this 'data' is from the decl, so not the pythonized data from pythonify.py
+        w_arr = space.call_obj_args(vector.clsdecl.get_overload('data'), w_vector, Arguments(space, []))
+        arr = space.interp_w(W_ArrayInstance, w_arr, can_be_None=True)
+        if not arr:
+            raise OperationError(space.w_StopIteration, space.w_None)
+
+        self.data    = rffi.cast(rffi.CCHARP, space.uint_w(arr.getbuffer(space)))
+        self.len     = space.uint_w(space.call_obj_args(vector.clsdecl.get_overload('size'), w_vector, Arguments(space, [])))
+        self.stride  = v_size
+
+    def descr_next(self, space):
+        if self.w_seq is None:
+            raise OperationError(space.w_StopIteration, space.w_None)
+        if self.len <= self.index:
+            self.w_seq = None
+            raise OperationError(space.w_StopIteration, space.w_None)
+        offset = lltype.direct_ptradd(self.data, rffi.cast(rffi.SIZE_T, self.index*self.stride))
+        w_item = self.converter.from_memory(space, space.w_None, rffi.cast(rffi.LONG, offset))
+        self.index += 1
+        return w_item
+
+def stdvector_iter(space, w_self):
+    return W_STLVectorIter(space, w_self)
+
+
 # setup pythonizations for later use at run-time
 _pythonizations = {}
 def register_pythonizations(space):
@@ -678,6 +774,12 @@
         ### std::string
         stdstring_c_str,
 
+        ### std::vector
+        stdvector_iter,
+
+        ### std::vector<bool>
+        vectorbool_getitem,
+        vectorbool_setitem,
     ]
 
     for f in allfuncs:
@@ -692,3 +794,10 @@
         space.setattr(w_pycppclass, space.newtext("c_str"), _pythonizations["stdstring_c_str"])
         _method_alias(space, w_pycppclass, "_cppyy_as_builtin", "c_str")
         _method_alias(space, w_pycppclass, "__str__",           "c_str")
+
+    if  name.find("std::vector<bool", 0, 16) == 0:
+        space.setattr(w_pycppclass, space.newtext("__getitem__"), _pythonizations["vectorbool_getitem"])
+        space.setattr(w_pycppclass, space.newtext("__setitem__"), _pythonizations["vectorbool_setitem"])
+
+    elif name.find("std::vector", 0, 11) == 0:
+        space.setattr(w_pycppclass, space.newtext("__iter__"), _pythonizations["stdvector_iter"])
diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -115,7 +115,7 @@
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         self._is_abstract(space)
 
     def to_memory(self, space, w_obj, w_value, offset):
@@ -142,7 +142,7 @@
         state = space.fromcache(ffitypes.State)
         return state.c_voidp
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         # read access, so no copy needed
         address_value = self._get_raw_address(space, w_obj, offset)
         address = rffi.cast(rffi.ULONG, address_value)
@@ -184,7 +184,7 @@
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'o'
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         # read access, so no copy needed
         address_value = self._get_raw_address(space, w_obj, offset)
         address = rffi.cast(rffi.ULONGP, address_value)
@@ -211,10 +211,13 @@
         x[0] = self._unwrap_object(space, w_obj)
 
     def default_argument_libffi(self, space, address):
+        if not self.valid_default:
+            from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
+            raise FastCallNotPossible
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self.default
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = self._get_raw_address(space, w_obj, offset)
         rffiptr = rffi.cast(self.c_ptrtype, address)
         return self._wrap_object(space, rffiptr[0])
@@ -224,7 +227,7 @@
         rffiptr = rffi.cast(self.c_ptrtype, address)
         rffiptr[0] = self._unwrap_object(space, w_value)
 
-class ConstRefNumericTypeConverterMixin(NumericTypeConverterMixin):
+class ConstRefNumericTypeConverterMixin(object):
     _mixin_ = True
 
     def cffi_type(self, space):
@@ -283,7 +286,7 @@
         x = rffi.cast(rffi.LONGP, address)
         x[0] = self._unwrap_object(space, w_obj)
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = rffi.cast(rffi.CCHARP, self._get_raw_address(space, w_obj, offset))
         if address[0] == '\x01':
             return space.w_True
@@ -308,7 +311,7 @@
         x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = rffi.cast(rffi.CCHARP, self._get_raw_address(space, w_obj, offset))
         return space.newbytes(address[0])
 
@@ -321,59 +324,92 @@
     pass
 
 class FloatConverter(ffitypes.typeid(rffi.FLOAT), FloatTypeConverterMixin, TypeConverter):
-    _immutable_fields_ = ['default']
+    _immutable_fields_ = ['default', 'valid_default']
 
     def __init__(self, space, default):
-        if default:
+        self.valid_default = False
+        try:
             fval = float(rfloat.rstring_to_float(default))
-        else:
+            self.valid_default = True
+        except Exception:
             fval = float(0.)
-        self.default = r_singlefloat(fval)
+        self.default = rffi.cast(rffi.FLOAT, r_singlefloat(fval))
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = self._get_raw_address(space, w_obj, offset)
         rffiptr = rffi.cast(self.c_ptrtype, address)
         return self._wrap_object(space, rffiptr[0])
 
-class ConstFloatRefConverter(FloatConverter):
+class ConstFloatRefConverter(ConstRefNumericTypeConverterMixin, FloatConverter):
     _immutable_fields_ = ['typecode']
     typecode = 'f'
 
-    def cffi_type(self, space):
-        state = space.fromcache(ffitypes.State)
-        return state.c_voidp
-
-    def convert_argument_libffi(self, space, w_obj, address, scratch):
-        from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
-        raise FastCallNotPossible
-
 class DoubleConverter(ffitypes.typeid(rffi.DOUBLE), FloatTypeConverterMixin, TypeConverter):
-    _immutable_fields_ = ['default']
+    _immutable_fields_ = ['default', 'valid_default']
 
     def __init__(self, space, default):
-        if default:
+        self.valid_default = False
+        try:
             self.default = rffi.cast(self.c_type, rfloat.rstring_to_float(default))
-        else:
+            self.valid_default = True
+        except Exception:
             self.default = rffi.cast(self.c_type, 0.)
 
 class ConstDoubleRefConverter(ConstRefNumericTypeConverterMixin, DoubleConverter):
     _immutable_fields_ = ['typecode']
     typecode = 'd'
 
-class LongDoubleConverter(ffitypes.typeid(rffi.LONGDOUBLE), FloatTypeConverterMixin, TypeConverter):
-    _immutable_fields_ = ['default']
+class LongDoubleConverter(TypeConverter):
+    _immutable_fields_ = ['default', 'valid_default']
+    typecode = 'g'
 
     def __init__(self, space, default):
-        if default:
-            fval = float(rfloat.rstring_to_float(default))
-        else:
-            fval = float(0.)
-        self.default = r_longfloat(fval)
+        self.valid_default = False
+        try:
+            # use float() instead of cast with r_longfloat
+            fval = rffi.cast(rffi.DOUBLE, rfloat.rstring_to_float(default))
+            self.valid_default = True
+        except Exception:
+            fval = rffi.cast(rffi.DOUBLE, 0.)
+        #self.default = r_longfloat(fval)
+        self.default = fval
+
+    def convert_argument(self, space, w_obj, address):
+        x = rffi.cast(rffi.VOIDP, address)
+        capi.c_double2longdouble(space, space.float_w(w_obj), x)
+        ba = rffi.cast(rffi.CCHARP, address)
+        ba[capi.c_function_arg_typeoffset(space)] = self.typecode
+
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
+        x = rffi.cast(rffi.VOIDP, address)
+        capi.c_double2longdouble(space, space.float_w(w_obj), x)
+
+    def default_argument_libffi(self, space, address):
+        if not self.valid_default:
+            from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
+            raise FastCallNotPossible
+        x = rffi.cast(rffi.VOIDP, address)
+        capi.c_double2longdouble(space, self.default, x)
+
+    def from_memory(self, space, w_obj, offset):
+        address = self._get_raw_address(space, w_obj, offset)
+        rffiptr = rffi.cast(rffi.VOIDP, address)
+        return space.newfloat(capi.c_longdouble2double(space, rffiptr))
+
+    def to_memory(self, space, w_obj, w_value, offset):
+        address = self._get_raw_address(space, w_obj, offset)
+        rffiptr = rffi.cast(rffi.VOIDP, address)
+        capi.c_double2longdouble(space, space.float_w(w_value), rffiptr)
 
 class ConstLongDoubleRefConverter(ConstRefNumericTypeConverterMixin, LongDoubleConverter):
     _immutable_fields_ = ['typecode']
     typecode = 'g'
 
+    def convert_argument_libffi(self, space, w_obj, address, scratch):
+        capi.c_double2longdouble(space, space.float_w(w_obj), rffi.cast(rffi.VOIDP, scratch))
+        x = rffi.cast(rffi.VOIDPP, address)
+        x[0] = scratch
+
 
 class CStringConverter(TypeConverter):
     def convert_argument(self, space, w_obj, address):
@@ -383,7 +419,7 @@
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = 'p'
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = self._get_raw_address(space, w_obj, offset)
         charpptr = rffi.cast(rffi.CCHARPP, address)
         return space.newtext(rffi.charp2str(charpptr[0]))
@@ -397,13 +433,15 @@
     def __init__(self, space, extra):
         self.size = extra
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = self._get_raw_address(space, w_obj, offset)
         charpptr = rffi.cast(rffi.CCHARP, address)
-        strsize = self.size
-        if charpptr[self.size-1] == '\0':
-            strsize = self.size-1  # rffi will add \0 back
-        return space.newtext(rffi.charpsize2str(charpptr, strsize))
+        if 0 <= self.size and self.size != 2**31-1:   # cling's code for "unknown" (?)
+            strsize = self.size
+            if charpptr[self.size-1] == '\0':
+                strsize = self.size-1  # rffi will add \0 back
+            return space.newtext(rffi.charpsize2str(charpptr, strsize))
+        return space.newtext(rffi.charp2str(charpptr))
 
 
 class VoidPtrConverter(TypeConverter):
@@ -428,7 +466,7 @@
         x = rffi.cast(rffi.VOIDPP, address)
         x[0] = self._unwrap_object(space, w_obj)
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         # returned as a long value for the address (INTPTR_T is not proper
         # per se, but rffi does not come with a PTRDIFF_T)
         address = self._get_raw_address(space, w_obj, offset)
@@ -540,7 +578,7 @@
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible       # TODO: by-value is a jit_libffi special case
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
         from pypy.module._cppyy import interp_cppyy
         return interp_cppyy.wrap_cppinstance(space, address, self.clsdecl, do_cast=False)
@@ -560,7 +598,7 @@
                 return capi.C_NULL_OBJECT
             raise e
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
         from pypy.module._cppyy import interp_cppyy
         return interp_cppyy.wrap_cppinstance(space, address, self.clsdecl, do_cast=False)
@@ -586,7 +624,7 @@
         from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
         from pypy.module._cppyy import interp_cppyy
         return interp_cppyy.wrap_cppinstance(
@@ -620,6 +658,26 @@
             lltype.free(self.ref_buffer, flavor='raw')
             self.ref_buffer = lltype.nullptr(rffi.VOIDPP.TO)
 
+class InstanceArrayConverter(InstancePtrConverter):
+    _immutable_fields_ = ['size']
+
+    def __init__(self, space, clsdecl, array_size, dimensions):
+        InstancePtrConverter.__init__(self, space, clsdecl)
+        if array_size <= 0 or array_size == 2**31-1:   # cling's code for "unknown" (?)
+            self.size = sys.maxint
+        else:
+            self.size = array_size
+        # peel one off as that should be the same as the array size
+        self.dimensions = dimensions[1:]
+
+    def from_memory(self, space, w_obj, offset):
+        address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
+        return lowlevelviews.W_ArrayOfInstances(space, self.clsdecl, address, self.size, self.dimensions)
+
+    def to_memory(self, space, w_obj, w_value, offset):
+        self._is_abstract(space)
+
+
 class StdStringConverter(InstanceConverter):
     def __init__(self, space, extra):
         from pypy.module._cppyy import interp_cppyy
@@ -790,7 +848,7 @@
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset(space)] = self.typecode
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
         from pypy.module._cppyy import interp_cppyy
         return interp_cppyy.wrap_cppinstance(space, address,
@@ -799,7 +857,7 @@
 class SmartPtrPtrConverter(SmartPtrConverter):
     typecode    = 'o'
 
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         self._is_abstract(space)
 
     def to_memory(self, space, w_obj, w_value, offset):
@@ -811,7 +869,7 @@
 
 
 class MacroConverter(TypeConverter):
-    def from_memory(self, space, w_obj, w_pycppclass, offset):
+    def from_memory(self, space, w_obj, offset):
         # TODO: get the actual type info from somewhere ...
         address = self._get_raw_address(space, w_obj, offset)
         longptr = rffi.cast(rffi.LONGP, address)
@@ -844,20 +902,28 @@
         pass
 
     # match of decorated, unqualified type
-    compound = helper.compound(name)
+    cpd = helper.compound(name)
     clean_name = capi.c_resolve_name(space, helper.clean_type(name))
     try:
-        return _converters[clean_name+compound](space, default)
+        return _converters[clean_name+cpd](space, default)
     except KeyError:
         pass
 
-    # arrays
+    # arrays (array_size may be negative, meaning: no size or no size found)
+    array_size = -1
+    if cpd == "[]":
+        array_size = helper.array_size(_name)    # uses original arg
+    elif cpd == '*' and ':' in default:
+        # this happens for multi-dimensional arrays: those are described as pointers
+        cpd = "[]"
+        splitpos = default.find(':')
+        if 0 < splitpos:     # always true, but needed for annotator
+            array_size = int(default[:splitpos])
+
     try:
-        # array_index may be negative to indicate no size or no size found
-        array_size = helper.array_size(_name)     # uses original arg
         # TODO: using clean_name here drops const (e.g. const char[] will
         # never be seen this way)
-        return _a_converters[clean_name+compound](space, array_size)
+        return _a_converters[clean_name+cpd](space, array_size)
     except KeyError:
         pass
 
@@ -871,24 +937,28 @@
         # check smart pointer type
         check_smart = capi.c_smartptr_info(space, clean_name)
         if check_smart[0]:
-            if compound == '':
+            if cpd == '':
                 return SmartPtrConverter(space, clsdecl, check_smart[1], check_smart[2])
-            elif compound == '*':
+            elif cpd == '*':
                 return SmartPtrPtrConverter(space, clsdecl, check_smart[1], check_smart[2])
-            elif compound == '&':
+            elif cpd == '&':
                 return SmartPtrRefConverter(space, clsdecl, check_smart[1], check_smart[2])
             # fall through: can still return smart pointer in non-smart way
 
         # type check for the benefit of the annotator
-        if compound == "*":
+        if cpd == "*":
             return InstancePtrConverter(space, clsdecl)
-        elif compound == "&":
+        elif cpd == "&":
             return InstanceRefConverter(space, clsdecl)
-        elif compound == "&&":
+        elif cpd == "&&":
             return InstanceMoveConverter(space, clsdecl)
-        elif compound == "**":
+        elif cpd in ["**", "*[]", "&*"]:
             return InstancePtrPtrConverter(space, clsdecl)
-        elif compound == "":
+        elif cpd == "[]" and array_size > 0:
+            # default encodes the dimensions
+            dims = default.split(':')
+            return InstanceArrayConverter(space, clsdecl, array_size, dims)
+        elif cpd == "":
             return InstanceConverter(space, clsdecl)
     elif "(anonymous)" in name:
         # special case: enum w/o a type name
@@ -900,7 +970,7 @@
             return FunctionPointerConverter(space, name[pos+2:])
 
     # void* or void converter (which fails on use)
-    if 0 <= compound.find('*'):
+    if 0 <= cpd.find('*'):
         return VoidPtrConverter(space, default)  # "user knows best"
 
     # return a void converter here, so that the class can be build even
@@ -915,8 +985,8 @@
 _converters["const float&"]             = ConstFloatRefConverter
 _converters["double"]                   = DoubleConverter
 _converters["const double&"]            = ConstDoubleRefConverter
-#_converters["long double"]              = LongDoubleConverter
-#_converters["const long double&"]       = ConstLongDoubleRefConverter
+_converters["long double"]              = LongDoubleConverter
+_converters["const long double&"]       = ConstLongDoubleRefConverter
 _converters["const char*"]              = CStringConverter
 _converters["void*"]                    = VoidPtrConverter
 _converters["void**"]                   = VoidPtrPtrConverter
@@ -950,7 +1020,12 @@
             _immutable_ = True
             typecode = c_tc
             def __init__(self, space, default):
-                self.default = rffi.cast(self.c_type, capi.c_strtoll(space, default))
+                self.valid_default = False
+                try:
+                    self.default = rffi.cast(self.c_type, capi.c_strtoll(space, default))
+                    self.valid_default = True
+                except Exception:
+                    self.default = rffi.cast(self.c_type, 0)
         class ConstRefConverter(ConstRefNumericTypeConverterMixin, BasicConverter):
             _immutable_ = True
         for name in names:
@@ -967,7 +1042,12 @@
             _immutable_ = True
             typecode = c_tc
             def __init__(self, space, default):
-                self.default = rffi.cast(self.c_type, capi.c_strtoll(space, default))
+                self.valid_default = False
+                try:
+                    self.default = rffi.cast(self.c_type, capi.c_strtoll(space, default))
+                    self.valid_default = True
+                except Exception:
+                    self.default = rffi.cast(self.c_type, 0)
         class ConstRefConverter(ConstRefNumericTypeConverterMixin, BasicConverter):
             _immutable_ = True
         for name in names:
@@ -987,7 +1067,12 @@
             _immutable_ = True
             typecode = c_tc
             def __init__(self, space, default):
-                self.default = rffi.cast(self.c_type, capi.c_strtoull(space, default))
+                self.valid_default = False
+                try:
+                    self.default = rffi.cast(self.c_type, capi.c_strtoull(space, default))
+                    self.valid_default = True
+                except Exception:
+                    self.default = rffi.cast(self.c_type, 0)
         class ConstRefConverter(ConstRefNumericTypeConverterMixin, BasicConverter):
             _immutable_ = True
         for name in names:
diff --git a/pypy/module/_cppyy/executor.py b/pypy/module/_cppyy/executor.py
--- a/pypy/module/_cppyy/executor.py
+++ b/pypy/module/_cppyy/executor.py
@@ -76,9 +76,6 @@
 class NumericExecutorMixin(object):
     _mixin_ = True
 
-    #def _wrap_object(self, space, obj):
-    #    return getattr(space, self.wrapper)(obj)
-
     def execute(self, space, cppmethod, cppthis, num_args, args):
         result = self.c_stubcall(space, cppmethod, cppthis, num_args, args)
         return self._wrap_object(space, rffi.cast(self.c_type, result))
@@ -100,13 +97,10 @@
         self.w_item = w_item
         self.do_assign = True
 
-    #def _wrap_object(self, space, obj):
-    #    return getattr(space, self.wrapper)(rffi.cast(self.c_type, obj))
-
     def _wrap_reference(self, space, rffiptr):
         if self.do_assign:
             rffiptr[0] = rffi.cast(self.c_type, self._unwrap_object(space, self.w_item))
-        self.do_assign = False
+            self.do_assign = False
         return self._wrap_object(space, rffiptr[0])    # all paths, for rtyper
 
     def execute(self, space, cppmethod, cppthis, num_args, args):
@@ -119,6 +113,48 @@
         return self._wrap_reference(space,
             rffi.cast(self.c_ptrtype, rffi.cast(rffi.VOIDPP, result)[0]))
 
+class LongDoubleExecutorMixin(object):
+    # Note: not really supported, but returns normal double
+    _mixin_ = True
+
+    def execute(self, space, cppmethod, cppthis, num_args, args):
+        result = self.c_stubcall(space, cppmethod, cppthis, num_args, args)
+        return space.newfloat(result)
+
+    def execute_libffi(self, space, cif_descr, funcaddr, buffer):
+        from pypy.module._cppyy.interp_cppyy import FastCallNotPossible
+        raise FastCallNotPossible
+
+class LongDoubleExecutor(ffitypes.typeid(rffi.LONGDOUBLE), LongDoubleExecutorMixin, Executor):
+    _immutable_ = True
+    c_stubcall  = staticmethod(capi.c_call_ld)
+
+class LongDoubleRefExecutorMixin(NumericRefExecutorMixin):
+    # Note: not really supported, but returns normal double
+    _mixin_ = True
+
+    def _wrap_reference(self, space, rffiptr):
+        if self.do_assign:
+            capi.c_double2longdouble(space, space.float_w(self.w_item), rffiptr)
+            self.do_assign = False
+            return self.w_item
+        return space.newfloat(capi.c_longdouble2double(space, rffiptr))
+
+    def execute(self, space, cppmethod, cppthis, num_args, args):
+        result = capi.c_call_r(space, cppmethod, cppthis, num_args, args)
+        return self._wrap_reference(space, rffi.cast(self.c_ptrtype, result))
+
+    def execute_libffi(self, space, cif_descr, funcaddr, buffer):
+        jit_libffi.jit_ffi_call(cif_descr, funcaddr, buffer)
+        result = rffi.ptradd(buffer, cif_descr.exchange_result)
+        return self._wrap_reference(space,
+            rffi.cast(self.c_ptrtype, rffi.cast(rffi.VOIDPP, result)[0]))
+
+class LongDoubleRefExecutor(ffitypes.typeid(rffi.LONGDOUBLE), LongDoubleRefExecutorMixin, Executor):
+    def cffi_type(self, space):
+        state = space.fromcache(ffitypes.State)
+        return state.c_voidp
+
 
 class CStringExecutor(Executor):
     def execute(self, space, cppmethod, cppthis, num_args, args):
@@ -345,6 +381,10 @@
 _executors["void*"]               = PtrTypeExecutor
 _executors["const char*"]         = CStringExecutor
 
+# long double not really supported: narrows to double
+_executors["long double"]          = LongDoubleExecutor
+_executors["long double&"]         = LongDoubleRefExecutor
+
 # special cases (note: 'string' aliases added below)
 _executors["constructor"]         = ConstructorExecutor
 
diff --git a/pypy/module/_cppyy/ffitypes.py b/pypy/module/_cppyy/ffitypes.py
--- a/pypy/module/_cppyy/ffitypes.py
+++ b/pypy/module/_cppyy/ffitypes.py
@@ -296,7 +296,8 @@
     _immutable_fields_ = ['c_type', 'c_ptrtype', 'typecode']
 
     c_type      = rffi.LONGDOUBLE
-    c_ptrtype   = rffi.LONGDOUBLEP
+    # c_ptrtype   = rffi.LONGDOUBLEP   # useless type at this point
+    c_ptrtype   = rffi.VOIDP
     typecode    = 'g'
 
     # long double is not really supported ...
@@ -304,7 +305,7 @@
         return r_longfloat(space.float_w(w_obj))
 
     def _wrap_object(self, space, obj):
-        return space.wrap(obj)
+        return space.newfloat(obj)
 
     def cffi_type(self, space):
         state = space.fromcache(State)
diff --git a/pypy/module/_cppyy/helper.py b/pypy/module/_cppyy/helper.py
--- a/pypy/module/_cppyy/helper.py
+++ b/pypy/module/_cppyy/helper.py
@@ -117,16 +117,10 @@
     # TODO: perhaps absorb or "pythonify" these operators?
     return cppname
 
-if sys.hexversion < 0x3000000:
-    CPPYY__div__  = "__div__"
-    CPPYY__idiv__ = "__idiv__"
-    CPPYY__long__ = "__long__"
-    CPPYY__bool__ = "__nonzero__"
-else:
-    CPPYY__div__  = "__truediv__"
-    CPPYY__idiv__ = "__itruediv__"
-    CPPYY__long__ = "__int__"
-    CPPYY__bool__ = "__bool__"
+CPPYY__div__  = "__div__"
+CPPYY__idiv__ = "__idiv__"
+CPPYY__long__ = "__long__"
+CPPYY__bool__ = "__nonzero__"
 
 # _operator_mappings["[]"]  = "__setitem__"      # depends on return type
 # _operator_mappings["+"]   = "__add__"          # depends on # of args (see __pos__)
diff --git a/pypy/module/_cppyy/include/capi.h b/pypy/module/_cppyy/include/capi.h
--- a/pypy/module/_cppyy/include/capi.h
+++ b/pypy/module/_cppyy/include/capi.h
@@ -63,6 +63,8 @@
     double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
     RPY_EXTERN
     long double cppyy_call_ld(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
+    double cppyy_call_nld(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
 
     RPY_EXTERN
     void* cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
@@ -151,11 +153,15 @@
     RPY_EXTERN
     char* cppyy_method_signature(cppyy_method_t, int show_formalargs);
     RPY_EXTERN
-    char* cppyy_method_prototype(cppyy_scope_t scope, cppyy_method_t idx, int show_formalargs);
+    char* cppyy_method_prototype(cppyy_scope_t scope, cppyy_method_t, int show_formalargs);
     RPY_EXTERN
     int cppyy_is_const_method(cppyy_method_t);
 
     RPY_EXTERN
+    int get_num_templated_methods(cppyy_scope_t scope);
+    RPY_EXTERN
+    char* get_templated_method_name(cppyy_scope_t scope, cppyy_index_t imeth);
+    RPY_EXTERN
     int cppyy_exists_method_template(cppyy_scope_t scope, const char* name);
     RPY_EXTERN
     int cppyy_method_is_template(cppyy_scope_t scope, cppyy_index_t idx);
@@ -216,9 +222,14 @@
     cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr);
 
     RPY_EXTERN
-    const char* cppyy_stdvector_valuetype(const char* clname);
+    double cppyy_longdouble2double(void*);
     RPY_EXTERN
-    size_t      cppyy_stdvector_valuesize(const char* clname);
+    void   cppyy_double2longdouble(double, void*);
+
+    RPY_EXTERN
+    int         cppyy_vectorbool_getitem(cppyy_object_t ptr, int idx);
+    RPY_EXTERN
+    void        cppyy_vectorbool_setitem(cppyy_object_t ptr, int idx, int value);
 
 #ifdef __cplusplus
 }
diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -24,6 +24,7 @@
 INSTANCE_FLAGS_IS_RVALUE   = 0x0004
 
 OVERLOAD_FLAGS_USE_FFI     = 0x0001
+OVERLOAD_FLAGS_CREATES     = 0x0002
 
 FUNCTION_IS_GLOBAL         = 0x0001
 FUNCTION_IS_STATIC         = 0x0001
@@ -229,8 +230,10 @@
         if self.converters is None:
             try:
                 self._setup(cppthis)
-            except Exception:
-                pass
+            except Exception as e:
+                if self.converters is None:
+                    raise oefmt(self.space.w_SystemError,
+                        "unable to initialize converters (%s)", str(e))
 
         # attempt to call directly through ffi chain
         if useffi and self._funcaddr:
@@ -458,6 +461,36 @@
 # need forwarding, which the normal instancemethod does not provide, hence this
 # derived class.
 class MethodWithProps(Method):
+    # set life management of result from the call
+    def fget_creates(self, space):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        return f.fget_creates(space)
+
+    @unwrap_spec(value=bool)
+    def fset_creates(self, space, value):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        f.fset_creates(space, value)
+
+    # set ownership policy of arguments (not yet implemented)
+    def fget_mempolicy(self, space):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        return f.fget_mempolicy(space)
+
+    @unwrap_spec(value=int)
+    def fset_mempolicy(self, space, value):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        f.fset_mempolicy(space, value)
+
+    # set to release the gil during call (not yet implemented)
+    def fget_release_gil(self, space):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        return f.fget_release_gil(space)
+
+    @unwrap_spec(value=bool)
+    def fset_release_gil(self, space, value):
+        f = space.interp_w(W_CPPOverload, self.w_function)
+        f.fset_release_gil(space, value)
+
     # allow user to determine ffi use rules per overload
     def fget_useffi(self, space):
         f = space.interp_w(W_CPPOverload, self.w_function)
@@ -473,19 +506,22 @@
     __doc__ = """cpp_instancemethod(function, instance, class)
 
 Create an instance method object.""",
-    __new__ = interp2app(MethodWithProps.descr_method__new__.im_func),
-    __call__ = interp2app(MethodWithProps.descr_method_call),
-    __get__ = interp2app(MethodWithProps.descr_method_get),
-    __func__ = interp_attrproperty_w('w_function', cls=MethodWithProps),
-    __self__ = interp_attrproperty_w('w_instance', cls=MethodWithProps),
+    __new__          = interp2app(MethodWithProps.descr_method__new__.im_func),
+    __call__         = interp2app(MethodWithProps.descr_method_call),
+    __get__          = interp2app(MethodWithProps.descr_method_get),
+    __func__         = interp_attrproperty_w('w_function', cls=MethodWithProps),
+    __self__         = interp_attrproperty_w('w_instance', cls=MethodWithProps),
     __getattribute__ = interp2app(MethodWithProps.descr_method_getattribute),
-    __eq__ = interp2app(MethodWithProps.descr_method_eq),
-    __ne__ = descr_generic_ne,
-    __hash__ = interp2app(MethodWithProps.descr_method_hash),
-    __repr__ = interp2app(MethodWithProps.descr_method_repr),
-    __reduce__ = interp2app(MethodWithProps.descr_method__reduce__),
-    __weakref__ = make_weakref_descr(MethodWithProps),
-    __useffi__ = GetSetProperty(MethodWithProps.fget_useffi, MethodWithProps.fset_useffi),
+    __eq__           = interp2app(MethodWithProps.descr_method_eq),
+    __ne__           = descr_generic_ne,
+    __hash__         = interp2app(MethodWithProps.descr_method_hash),
+    __repr__         = interp2app(MethodWithProps.descr_method_repr),
+    __reduce__       = interp2app(MethodWithProps.descr_method__reduce__),
+    __weakref__      = make_weakref_descr(MethodWithProps),
+    __creates__      = GetSetProperty(MethodWithProps.fget_creates,     MethodWithProps.fset_creates),
+    __mempolicy__    = GetSetProperty(MethodWithProps.fget_mempolicy,   MethodWithProps.fset_mempolicy),
+    __release_gil__  = GetSetProperty(MethodWithProps.fget_release_gil, MethodWithProps.fset_release_gil),
+    __useffi__       = GetSetProperty(MethodWithProps.fget_useffi,      MethodWithProps.fset_useffi),
     )
 MethodWithProps.typedef.acceptable_as_base_class = False
 
@@ -548,7 +584,12 @@
         for i in range(len(self.functions)):
             cppyyfunc = self.functions[i]
             try:
-                return cppyyfunc.call(cppthis, args_w, self.flags & OVERLOAD_FLAGS_USE_FFI)
+                w_result = cppyyfunc.call(cppthis, args_w, self.flags & OVERLOAD_FLAGS_USE_FFI)
+                if self.flags & OVERLOAD_FLAGS_CREATES:
+                    if isinstance(w_result, W_CPPInstance):
+                        cppinstance = self.space.interp_w(W_CPPInstance, w_result)
+                        cppinstance.fset_python_owns(self.space, self.space.w_True)
+                return w_result
             except Exception:
                 pass
 
@@ -561,6 +602,7 @@
         for i in range(len(self.functions)):
             cppyyfunc = self.functions[i]
             try:
+                # no need to set ownership on the return value, as none of the methods execute
                 return cppyyfunc.call(cppthis, args_w, self.flags & OVERLOAD_FLAGS_USE_FFI)
             except OperationError as e:
                 # special case if there's just one function, to prevent clogging the error message
@@ -599,6 +641,33 @@
                 return W_CPPOverload(self.space, self.scope, [f])
         raise oefmt(self.space.w_LookupError, "signature '%s' not found", signature)
 
+    # set life management of result from the call
+    def fget_creates(self, space):
+        return space.newbool(bool(self.flags & OVERLOAD_FLAGS_CREATES))
+
+    @unwrap_spec(value=bool)
+    def fset_creates(self, space, value):
+        if space.is_true(value):
+            self.flags |= OVERLOAD_FLAGS_CREATES
+        else:
+            self.flags &= ~OVERLOAD_FLAGS_CREATES
+
+    # set ownership policy of arguments (not yet implemented)
+    def fget_mempolicy(self, space):
+        return space.newint(0)
+
+    @unwrap_spec(value=int)
+    def fset_mempolicy(self, space, value):
+        pass
+
+    # set to release the gil during call (not yet implemented)
+    def fget_release_gil(self, space):
+        return space.newbool(True)
+
+    @unwrap_spec(value=bool)
+    def fset_release_gil(self, space, value):
+        pass
+
     # allow user to determine ffi use rules per overload
     def fget_useffi(self, space):
         return space.newbool(bool(self.flags & OVERLOAD_FLAGS_USE_FFI))
@@ -622,11 +691,14 @@
 
 W_CPPOverload.typedef = TypeDef(
     'CPPOverload',
-    __get__      = interp2app(W_CPPOverload.descr_get),
-    __call__     = interp2app(W_CPPOverload.call_args),
-    __useffi__   = GetSetProperty(W_CPPOverload.fget_useffi, W_CPPOverload.fset_useffi),
-    __overload__ = interp2app(W_CPPOverload.mp_overload),
-    __doc__      = GetSetProperty(W_CPPOverload.fget_doc)
+    __get__         = interp2app(W_CPPOverload.descr_get),
+    __call__        = interp2app(W_CPPOverload.call_args),
+    __creates__     = GetSetProperty(W_CPPOverload.fget_creates,     W_CPPOverload.fset_creates),
+    __mempolicy__   = GetSetProperty(W_CPPOverload.fget_mempolicy,   W_CPPOverload.fset_mempolicy),
+    __release_gil__ = GetSetProperty(W_CPPOverload.fget_release_gil, W_CPPOverload.fset_release_gil),
+    __useffi__      = GetSetProperty(W_CPPOverload.fget_useffi,      W_CPPOverload.fset_useffi),
+    __overload__    = interp2app(W_CPPOverload.mp_overload),
+    __doc__         = GetSetProperty(W_CPPOverload.fget_doc)
 )
 
 
@@ -655,11 +727,14 @@
 
 W_CPPStaticOverload.typedef = TypeDef(
     'CPPStaticOverload',
-    __get__      = interp2app(W_CPPStaticOverload.descr_get),
-    __call__     = interp2app(W_CPPStaticOverload.call_args),
-    __useffi__   = GetSetProperty(W_CPPStaticOverload.fget_useffi, W_CPPStaticOverload.fset_useffi),
-    __overload__ = interp2app(W_CPPStaticOverload.mp_overload),
-    __doc__      = GetSetProperty(W_CPPStaticOverload.fget_doc)
+    __get__         = interp2app(W_CPPStaticOverload.descr_get),
+    __call__        = interp2app(W_CPPStaticOverload.call_args),
+    __creates__     = GetSetProperty(W_CPPStaticOverload.fget_creates,     W_CPPStaticOverload.fset_creates),
+    __mempolicy__   = GetSetProperty(W_CPPStaticOverload.fget_mempolicy,   W_CPPStaticOverload.fset_mempolicy),
+    __release_gil__ = GetSetProperty(W_CPPStaticOverload.fget_release_gil, W_CPPStaticOverload.fset_release_gil),
+    __useffi__      = GetSetProperty(W_CPPStaticOverload.fget_useffi,      W_CPPStaticOverload.fset_useffi),
+    __overload__    = interp2app(W_CPPStaticOverload.mp_overload),
+    __doc__         = GetSetProperty(W_CPPStaticOverload.fget_doc)
 )
 
 
@@ -712,6 +787,7 @@
 class TemplateOverloadMixin(object):
     """Mixin to instantiate templated methods/functions."""
 
+    _attrs_ = ['tmpl_args_w']
     _mixin_ = True
 
     def construct_template_args(self, w_tpArgs, args_w = None):
@@ -764,23 +840,37 @@
         return cppol
 
     def instantiate_and_call(self, name, args_w):
-        # try to match with run-time instantiations
-        for cppol in self.master.overloads.values():
-            try:
-                if not self.space.is_w(self.w_this, self.space.w_None):
-                    return self.space.call_obj_args(cppol, self.w_this, Arguments(self.space, args_w))
-                return self.space.call_args(cppol, Arguments(self.space, args_w))
-            except Exception:
-                pass    # completely ignore for now; have to see whether errors become confusing
+        method = None
+        try:
+            # existing cached instantiations
+            if name[-1] == '>':   # only accept full templated name, to ensure explicit
+                method = self.master.overloads[name]
+            else:
+            # try to match with run-time instantiations
+                # TODO: logically, this could be used, but in practice, it's proving too
+                #  greedy ... maybe as a last resort?
+                #for cppol in self.master.overloads.values():
+                #    try:
+                #        if not self.space.is_w(self.w_this, self.space.w_None):
+                #            return self.space.call_obj_args(cppol, self.w_this, Arguments(self.space, args_w))
+                #        return self.space.call_args(cppol, Arguments(self.space, args_w))
+                #    except Exception:
+                #        pass    # completely ignore for now; have to see whether errors become confusing
+                raise TypeError("pre-existing overloads failed")
+        except (KeyError, TypeError):
+            # if not known, try to deduce from argument types
+            w_types = self.space.newtuple([self.space.type(obj_w) for obj_w in args_w])
+            proto = self.construct_template_args(w_types, args_w)
+            method = self.find_method_template(name, proto)
 
-        # if all failed, then try to deduce from argument types
-        w_types = self.space.newtuple([self.space.type(obj_w) for obj_w in args_w])
-        proto = self.construct_template_args(w_types, args_w)
-        method = self.find_method_template(name, proto)
-
-        # only cache result if the name retains the full template
-        fullname = capi.c_method_full_name(self.space, method.functions[0].cppmethod)
-        if 0 <= fullname.rfind('>'):
+            # only cache result if the name retains the full template
+            # TODO: the problem is in part that c_method_full_name returns incorrect names,
+            # e.g. when default template arguments are involved, so for now use 'name' if it
+            # has the full templated name
+            if name[-1] == '>':
+                fullname = name
+            else:
+                fullname = capi.c_method_full_name(self.space, method.functions[0].cppmethod)
             try:
                 existing = self.master.overloads[fullname]
                 allf = existing.functions + method.functions
@@ -792,9 +882,10 @@
             except KeyError:
                 self.master.overloads[fullname] = method
 
-        if not self.space.is_w(self.w_this, self.space.w_None):
-            return self.space.call_obj_args(method, self.w_this, Arguments(self.space, args_w))
-        return self.space.call_args(method, Arguments(self.space, args_w))
+        if method is not None:
+            if not self.space.is_w(self.w_this, self.space.w_None):
+                return self.space.call_obj_args(method, self.w_this, Arguments(self.space, args_w))
+            return self.space.call_args(method, Arguments(self.space, args_w))
 
     def getitem_impl(self, name, args_w):
         space = self.space
@@ -808,14 +899,9 @@
         fullname = name+'<'+tmpl_args+'>'
         try:
             method = self.master.overloads[fullname]
-        except KeyError:
-            method = self.find_method_template(fullname)
-            # cache result (name is always full templated name)
-            self.master.overloads[fullname] = method
-            # also cache on "official" name (may include default template arguments)
-            c_fullname = capi.c_method_full_name(self.space, method.functions[0].cppmethod)
-            if c_fullname != fullname:
-                self.master.overloads[c_fullname] = method
+        except KeyError as e:
+            # defer instantiation until arguments are known
+            return self.clone(tmpl_args)
 
         return method.descr_get(self.w_this, None)
 
@@ -824,21 +910,29 @@
     """App-level dispatcher to allow both lookup/instantiation of templated methods and
     dispatch among overloads between templated and non-templated method."""
 
-    _attrs_ = ['name', 'overloads', 'master', 'w_this']
-    _immutable_fields_ = ['name']
+    _attrs_ = ['name', 'tmpl_args', 'overloads', 'master', 'w_this']
+    _immutable_fields_ = ['name', 'tmpl_args']
 
-    def __init__(self, space, name, decl_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
+    def __init__(self, space, name, tmpl_args, decl_scope, functions, flags = OVERLOAD_FLAGS_USE_FFI):
          W_CPPOverload.__init__(self, space, decl_scope, functions, flags)
          self.name = name
+         self.tmpl_args = tmpl_args
          self.overloads = {}
          self.master = self
          self.w_this = space.w_None
 
+    def clone(self, tmpl_args):
+        other = W_CPPTemplateOverload(self.space, self.name, tmpl_args, self.scope, self.functions, self.flags)
+        other.overloads = self.overloads
+        other.master = self.master
+        other.w_this = self.w_this
+        return other
+
     def descr_get(self, w_cppinstance, w_cls=None):
         # TODO: don't return copy, but bind in an external object (like W_CPPOverload)
         if self.space.is_w(w_cppinstance, self.space.w_None):
             return self  # unbound, so no new instance needed
-        cppol = W_CPPTemplateOverload(self.space, self.name, self.scope, self.functions, self.flags)
+        cppol = W_CPPTemplateOverload(self.space, self.name, self.tmpl_args, self.scope, self.functions, self.flags)
         cppol.w_this = w_cppinstance
         cppol.master = self.master
         return cppol     # bound
@@ -848,13 +942,18 @@
         # direct call: either pick non-templated overload or attempt to deduce
         # the template instantiation from the argument types
 
-        # try existing overloads or compile-time instantiations
+        # do explicit lookup with tmpl_args if given
         try:
-            return W_CPPOverload.call_args(self, args_w)
+            fullname = self.name
+            if self.tmpl_args is not None:
+                fullname = fullname+'<'+self.tmpl_args+'>'
+            return self.instantiate_and_call(fullname, args_w)
         except Exception:
             pass
 
-        return self.instantiate_and_call(self.name, args_w)
+        # otherwise, try existing overloads or compile-time instantiations
+        # TODO: consolidate errors
+        return W_CPPOverload.call_args(self, [self.w_this]+args_w)
 
     @unwrap_spec(args_w='args_w')
     def getitem(self, args_w):
@@ -868,33 +967,44 @@
 
 W_CPPTemplateOverload.typedef = TypeDef(
     'CPPTemplateOverload',
-    __get__      = interp2app(W_CPPTemplateOverload.descr_get),
-    __getitem__  = interp2app(W_CPPTemplateOverload.getitem),
-    __call__     = interp2app(W_CPPTemplateOverload.call_args),
-    __useffi__   = GetSetProperty(W_CPPTemplateOverload.fget_useffi, W_CPPTemplateOverload.fset_useffi),
-    __doc__      = GetSetProperty(W_CPPTemplateOverload.fget_doc)
+    __get__         = interp2app(W_CPPTemplateOverload.descr_get),
+    __getitem__     = interp2app(W_CPPTemplateOverload.getitem),
+    __call__        = interp2app(W_CPPTemplateOverload.call_args),
+    __creates__     = GetSetProperty(W_CPPTemplateOverload.fget_creates,     W_CPPTemplateOverload.fset_creates),
+    __mempolicy__   = GetSetProperty(W_CPPTemplateOverload.fget_mempolicy,   W_CPPTemplateOverload.fset_mempolicy),
+    __release_gil__ = GetSetProperty(W_CPPTemplateOverload.fget_release_gil, W_CPPTemplateOverload.fset_release_gil),
+    __useffi__      = GetSetProperty(W_CPPTemplateOverload.fget_useffi,      W_CPPTemplateOverload.fset_useffi),
+    __doc__         = GetSetProperty(W_CPPTemplateOverload.fget_doc)
 )
 
 class W_CPPTemplateStaticOverload(W_CPPStaticOverload, TemplateOverloadMixin):
     """App-level dispatcher to allow both lookup/instantiation of templated methods and
     dispatch among overloads between templated and non-templated method."""
 
-    _attrs_ = ['name', 'overloads', 'master', 'w_this']
-    _immutable_fields_ = ['name']
+    _attrs_ = ['name', 'tmpl_args', 'overloads', 'master', 'w_this']
+    _immutable_fields_ = ['name', 'tmpl_args']
 
-    def __init__(self, space, name, decl_scope, funcs, flags = OVERLOAD_FLAGS_USE_FFI):
+    def __init__(self, space, name, tmpl_args, decl_scope, funcs, flags = OVERLOAD_FLAGS_USE_FFI):
          W_CPPStaticOverload.__init__(self, space, decl_scope, funcs, flags)
          self.name = name
+         self.tmpl_args = tmpl_args
          self.overloads = {}
          self.master = self
          self.w_this = space.w_None
 
+    def clone(self, tmpl_args):
+        other = W_CPPTemplateStaticOverload(self.space, self.name, tmpl_args, self.scope, self.functions, self.flags)
+        other.overloads = self.overloads
+        other.master = self.master
+        other.w_this = self.w_this
+        return other
+
     def descr_get(self, w_cppinstance, w_cls=None):
         # TODO: don't return copy, but bind in an external object (like W_CPPOverload)
         if isinstance(w_cppinstance, W_CPPInstance):
             cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance)
             if cppinstance.clsdecl.handle != self.scope.handle:
-                cppol = W_CPPTemplateStaticOverload(self.space, self.name, self.scope, self.functions, self.flags)
+                cppol = W_CPPTemplateStaticOverload(self.space, self.name, self.tmpl_args, self.scope, self.functions, self.flags)
                 cppol.w_this = w_cppinstance
                 cppol.master = self.master
                 return cppol       # bound
@@ -904,15 +1014,20 @@
     def call_args(self, args_w):
         # direct call: either pick non-templated overload or attempt to deduce
         # the template instantiation from the argument types
+        # TODO: refactor with W_CPPTemplateOverload
 
-        # try existing overloads or compile-time instantiations
+        # do explicit lookup with tmpl_args if given
         try:
-            return W_CPPStaticOverload.call_args(self, args_w)
+            fullname = self.name
+            if self.tmpl_args is not None:
+                fullname = fullname+'<'+self.tmpl_args+'>'
+            return self.instantiate_and_call(fullname, args_w)
         except Exception:
             pass
 
-        # try new instantiation
-        return self.instantiate_and_call(self.name, args_w)
+        # otherwise, try existing overloads or compile-time instantiations
+        # TODO: consolidate errors
+        return W_CPPStaticOverload.call_args(self, args_w)
 
     @unwrap_spec(args_w='args_w')
     def getitem(self, args_w):
@@ -926,11 +1041,18 @@
 
 W_CPPTemplateStaticOverload.typedef = TypeDef(
     'CPPTemplateStaticOverload',
-    __get__      = interp2app(W_CPPTemplateStaticOverload.descr_get),
-    __getitem__  = interp2app(W_CPPTemplateStaticOverload.getitem),
-    __call__     = interp2app(W_CPPTemplateStaticOverload.call_args),
-    __useffi__   = GetSetProperty(W_CPPTemplateStaticOverload.fget_useffi, W_CPPTemplateStaticOverload.fset_useffi),
-    __doc__      = GetSetProperty(W_CPPTemplateStaticOverload.fget_doc)
+    __get__         = interp2app(W_CPPTemplateStaticOverload.descr_get),
+    __getitem__     = interp2app(W_CPPTemplateStaticOverload.getitem),
+    __call__        = interp2app(W_CPPTemplateStaticOverload.call_args),
+    __creates__     = GetSetProperty(W_CPPTemplateStaticOverload.fget_creates,
+                                     W_CPPTemplateStaticOverload.fset_creates),
+    __mempolicy__   = GetSetProperty(W_CPPTemplateStaticOverload.fget_mempolicy,
+                                     W_CPPTemplateStaticOverload.fset_mempolicy),
+    __release_gil__ = GetSetProperty(W_CPPTemplateStaticOverload.fget_release_gil,
+                                     W_CPPTemplateStaticOverload.fset_release_gil),
+    __useffi__      = GetSetProperty(W_CPPTemplateStaticOverload.fget_useffi,
+                                     W_CPPTemplateStaticOverload.fset_useffi),
+    __doc__         = GetSetProperty(W_CPPTemplateStaticOverload.fget_doc)
 )
 
 
@@ -950,11 +1072,11 @@
     _attrs_ = ['space', 'scope', 'converter', 'offset']
     _immutable_fields = ['scope', 'converter', 'offset']
 
-    def __init__(self, space, decl_scope, type_name, offset):
+    def __init__(self, space, decl_scope, type_name, dimensions, offset):
         self.space = space
         self.scope = decl_scope
-        self.converter = converter.get_converter(self.space, type_name, '')
-        self.offset = offset
+        self.converter = converter.get_converter(self.space, type_name, dimensions)
+        self.offset = rffi.cast(rffi.LONG, offset)
 
     def _get_offset(self, cppinstance):
         if cppinstance:
@@ -971,7 +1093,7 @@
             raise oefmt(self.space.w_AttributeError,
                         "attribute access requires an instance")
         offset = self._get_offset(cppinstance)
-        return self.converter.from_memory(self.space, w_cppinstance, w_pycppclass, offset)
+        return self.converter.from_memory(self.space, w_cppinstance, offset)
 
     def set(self, w_cppinstance, w_value):
         cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=True)
@@ -1008,7 +1130,7 @@
         return self.offset
 
     def get(self, w_cppinstance, w_pycppclass):
-        return self.converter.from_memory(self.space, self.space.w_None, w_pycppclass, self.offset)
+        return self.converter.from_memory(self.space, self.space.w_None, self.offset)
 
     def set(self, w_cppinstance, w_value):
         self.converter.to_memory(self.space, self.space.w_None, w_value, self.offset)
@@ -1098,6 +1220,21 @@
         self.datamembers[name] = new_dm
         return new_dm
 
+    @unwrap_spec(name='text')
+    def has_enum(self, name):
+        if capi.c_is_enum(self.space, self.name+'::'+name):
+            return self.space.w_True
+        return self.space.w_False
+
+    def _encode_dm_dimensions(self, idata):
+        # encode dimensions (TODO: this is ugly, but here's where the info is)
+        dims = []
+        sz = capi.c_get_dimension_size(self.space, self, idata, len(dims))
+        while 0 < sz:
+            dims.append(str(sz))
+            sz = capi.c_get_dimension_size(self.space, self, idata, len(dims))
+        return ':'.join(dims)
+
     @unwrap_spec(name='text', signature='text')
     def scope__dispatch__(self, name, signature):
         overload = self.get_overload(name)
@@ -1137,13 +1274,16 @@
 
     def _make_datamember(self, dm_name, dm_idx):
         type_name = capi.c_datamember_type(self.space, self, dm_idx)
+        if capi.c_is_enum_data(self.space, self, dm_idx):
+            type_name = capi.c_resolve_enum(self.space, type_name)
         offset = capi.c_datamember_offset(self.space, self, dm_idx)
         if offset == -1:
             raise self.missing_attribute_error(dm_name)
+        dims = self._encode_dm_dimensions(dm_idx)
         if capi.c_is_const_data(self.space, self, dm_idx):
-            datamember = W_CPPConstStaticData(self.space, self, type_name, offset)
+            datamember = W_CPPConstStaticData(self.space, self, type_name, dims, offset)
         else:
-            datamember = W_CPPStaticData(self.space, self, type_name, offset)
+            datamember = W_CPPStaticData(self.space, self, type_name, dims, offset)
         self.datamembers[dm_name] = datamember
         return datamember
 
@@ -1158,10 +1298,10 @@
                 if capi.c_method_is_template(self.space, self, idx):
                     templated = True
             if templated:
-                return W_CPPTemplateStaticOverload(self.space, meth_name, self, cppfunctions[:])
+                return W_CPPTemplateStaticOverload(self.space, meth_name, None, self, cppfunctions[:])
             return W_CPPStaticOverload(self.space, self, cppfunctions[:])
         elif capi.c_exists_method_template(self.space, self, meth_name):
-            return W_CPPTemplateStaticOverload(self.space, meth_name, self, [])
+            return W_CPPTemplateStaticOverload(self.space, meth_name, None, self, [])
         raise self.missing_attribute_error(meth_name)
 
     def find_datamember(self, dm_name):
@@ -1193,6 +1333,7 @@
     get_datamember_names = interp2app(W_CPPNamespaceDecl.get_datamember_names),
     get_datamember = interp2app(W_CPPNamespaceDecl.get_datamember),
     is_namespace = interp2app(W_CPPNamespaceDecl.is_namespace),
+    has_enum = interp2app(W_CPPNamespaceDecl.has_enum),