[pypy-commit] pypy py3.5: merged default
plan_rich
pypy.commits at gmail.com
Mon Jan 9 05:04:53 EST 2017
Author: Richard Plangger <planrichi at gmail.com>
Branch: py3.5
Changeset: r89433:8f58de1af422
Date: 2017-01-09 11:01 +0100
http://bitbucket.org/pypy/pypy/changeset/8f58de1af422/
Log: merged default
diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py
--- a/lib-python/2.7/ctypes/test/test_frombuffer.py
+++ b/lib-python/2.7/ctypes/test/test_frombuffer.py
@@ -32,7 +32,7 @@
del a; gc.collect(); gc.collect(); gc.collect()
self.assertEqual(x[:], expected)
- self.assertRaises((TypeError, ValueError),
+ self.assertRaises(TypeError,
(c_char * 16).from_buffer, "a" * 16)
def test_fom_buffer_with_offset(self):
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -428,11 +428,6 @@
``datetime.date`` is the superclass of ``datetime.datetime``).
Anyway, the proper fix is arguably to use a regular method call in
the first place: ``datetime.date.today().strftime(...)``
-
-* the ``__dict__`` attribute of new-style classes returns a normal dict, as
- opposed to a dict proxy like in CPython. Mutating the dict will change the
- type and vice versa. For builtin types, a dictionary will be returned that
- cannot be changed (but still looks and behaves like a normal dictionary).
* some functions and attributes of the ``gc`` module behave in a
slightly different way: for example, ``gc.enable`` and
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -90,3 +90,18 @@
Support translations of cpyext with the Boehm GC (for special cases like
revdb).
+
+.. branch: strbuf-as-buffer
+
+Implement StringBuffer.get_raw_address (missing feature for the buffer protocol).
+More generally it is now possible to obtain the address of any object (if it
+is readonly) without pinning it.
+
+.. branch: cpyext-cleanup
+
+Refactor cpyext initialisation.
+
+.. branch: cpyext-from2
+
+Fix a test failure introduced by strbuf-as-buffer
+
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -662,9 +662,6 @@
def setup_builtin_modules(self):
"NOT_RPYTHON: only for initializing the space."
- if self.config.objspace.usemodules.cpyext:
- from pypy.module.cpyext.state import State
- self.fromcache(State).build_api(self)
self.getbuiltinmodule('sys')
self.getbuiltinmodule('_imp')
self.getbuiltinmodule('_frozen_importlib')
diff --git a/pypy/module/__pypy__/bytebuffer.py b/pypy/module/__pypy__/bytebuffer.py
--- a/pypy/module/__pypy__/bytebuffer.py
+++ b/pypy/module/__pypy__/bytebuffer.py
@@ -4,6 +4,7 @@
from rpython.rlib.buffer import Buffer
from pypy.interpreter.gateway import unwrap_spec
+from rpython.rlib.rgc import nonmoving_raw_ptr_for_resizable_list
class ByteBuffer(Buffer):
@@ -22,6 +23,8 @@
def setitem(self, index, char):
self.data[index] = char
+ def get_raw_address(self):
+ return nonmoving_raw_ptr_for_resizable_list(self.data)
@unwrap_spec(length=int)
def bytebuffer(space, length):
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -136,6 +136,9 @@
return _from_buffer(space, w_ctype, w_x)
def _from_buffer(space, w_ctype, w_x):
+ if space.isinstance_w(w_x, space.w_unicode):
+ raise oefmt(space.w_TypeError,
+ "from_buffer() cannot return the address a unicode")
buf = _fetch_as_read_buffer(space, w_x)
if space.isinstance_w(w_x, space.w_str):
_cdata = get_raw_address_of_string(space, w_x)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3427,16 +3427,26 @@
except ImportError:
pass
else:
- # from_buffer(buffer(b"foo")) does not work, because it's not
- # implemented on pypy; only from_buffer(b"foo") works.
- py.test.raises(TypeError, from_buffer, BCharA, buffer(b"foo"))
- py.test.raises(TypeError, from_buffer, BCharA, buffer(u+"foo"))
+ # Python 2 only
+ contents = from_buffer(BCharA, buffer(b"foo"))
+ assert len(contents) == len(p1)
+ for i in range(len(contents)):
+ assert contents[i] == p1[i]
+ p4 = buffer(u+"foo")
+ contents = from_buffer(BCharA, buffer(u+"foo"))
+ assert len(contents) == len(p4)
+ for i in range(len(contents)):
+ assert contents[i] == p4[i]
try:
from __builtin__ import memoryview
except ImportError:
pass
else:
- py.test.raises(TypeError, from_buffer, BCharA, memoryview(b"foo"))
+ contents = from_buffer(BCharA, memoryview(b"foo"))
+ assert len(contents) == len(p1)
+ for i in range(len(contents)):
+ assert contents[i] == p1[i]
+
def test_from_buffer_bytearray():
a = bytearray(b"xyz")
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -494,7 +494,7 @@
while True:
try:
addr = self.addr_from_object(space, w_addr)
- count = self.sock.sendto(data, flags, addr)
+ count = self.sock.sendto(data, len(data), flags, addr)
break
except SocketError as e:
converted_error(space, e, eintr_retry=True)
diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py
--- a/pypy/module/cpyext/__init__.py
+++ b/pypy/module/cpyext/__init__.py
@@ -1,5 +1,4 @@
from pypy.interpreter.mixedmodule import MixedModule
-from pypy.interpreter import gateway
from pypy.module.cpyext.state import State
from pypy.module.cpyext import api
@@ -13,12 +12,17 @@
atexit_funcs = []
+ def setup_after_space_initialization(self):
+ state = self.space.fromcache(State)
+ state.setup_rawrefcount()
+ state.build_api()
+
def startup(self, space):
space.fromcache(State).startup(space)
method = pypy.module.cpyext.typeobject.get_new_method_def(space)
w_obj = pypy.module.cpyext.methodobject.W_PyCFunctionObject(space, method, space.wrap(''))
space.appexec([space.type(w_obj)], """(methodtype):
- from pickle import Pickler
+ from pickle import Pickler
Pickler.dispatch[methodtype] = Pickler.save_global
""")
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1,5 +1,6 @@
import ctypes
import sys, os
+from collections import defaultdict
import py
@@ -70,7 +71,6 @@
class CConfig_constants:
_compilation_info_ = CConfig._compilation_info_
-VA_LIST_P = rffi.VOIDP # rffi.COpaquePtr('va_list')
CONST_STRING = lltype.Ptr(lltype.Array(lltype.Char,
hints={'nolength': True}),
use_cache=False)
@@ -367,8 +367,8 @@
func_name = func.func_name
if header is not None:
c_name = None
- assert func_name not in FUNCTIONS, (
- "%s already registered" % func_name)
+ if func_name in FUNCTIONS_BY_HEADER[header]:
+ raise ValueError("%s already registered" % func_name)
else:
c_name = func_name
api_function = ApiFunction(argtypes, restype, func, error,
@@ -466,9 +466,7 @@
return res
if header is not None:
- if header == DEFAULT_HEADER:
- FUNCTIONS[func_name] = api_function
- FUNCTIONS_BY_HEADER.setdefault(header, {})[func_name] = api_function
+ FUNCTIONS_BY_HEADER[header][func_name] = api_function
INTERPLEVEL_API[func_name] = unwrapper_catch # used in tests
return unwrapper # used in 'normal' RPython code.
return decorate
@@ -492,8 +490,7 @@
GLOBALS[name] = (typ, expr)
INTERPLEVEL_API = {}
-FUNCTIONS = {}
-FUNCTIONS_BY_HEADER = {}
+FUNCTIONS_BY_HEADER = defaultdict(dict)
# These are C symbols which cpyext will export, but which are defined in .c
# files somewhere in the implementation of cpyext (rather than being defined in
@@ -528,6 +525,8 @@
'PyCapsule_SetPointer', 'PyCapsule_SetName', 'PyCapsule_SetDestructor',
'PyCapsule_SetContext', 'PyCapsule_Import', 'PyCapsule_Type', '_Py_get_capsule_type',
+ 'PyComplex_AsCComplex', 'PyComplex_FromCComplex',
+
'PyObject_AsReadBuffer', 'PyObject_AsWriteBuffer', 'PyObject_CheckReadBuffer',
'PyOS_getsig', 'PyOS_setsig',
@@ -574,7 +573,9 @@
# PyExc_AttributeError, PyExc_OverflowError, PyExc_ImportError,
# PyExc_NameError, PyExc_MemoryError, PyExc_RuntimeError,
# PyExc_UnicodeEncodeError, PyExc_UnicodeDecodeError, ...
- for exc_name in exceptions.Module.interpleveldefs.keys():
+ global all_exceptions
+ all_exceptions = list(exceptions.Module.interpleveldefs)
+ for exc_name in all_exceptions:
if exc_name in ('EnvironmentError', 'IOError'):
# FIXME: aliases of OSError cause a clash of names via
# export_struct
@@ -619,12 +620,6 @@
% (cpyname, ))
build_exported_objects()
-def get_structtype_for_ctype(ctype):
- from pypy.module.cpyext.typeobjectdefs import PyTypeObjectPtr
- from pypy.module.cpyext.cdatetime import PyDateTime_CAPI
- return {"PyObject*": PyObject, "PyTypeObject*": PyTypeObjectPtr,
- "PyDateTime_CAPI*": lltype.Ptr(PyDateTime_CAPI)}[ctype]
-
# Note: as a special case, "PyObject" is the pointer type in RPython,
# corresponding to "PyObject *" in C. We do that only for PyObject.
# For example, "PyTypeObject" is the struct type even in RPython.
@@ -673,12 +668,6 @@
# a pointer to PyObject
PyObjectP = rffi.CArrayPtr(PyObject)
-VA_TP_LIST = {}
-#{'int': lltype.Signed,
-# 'PyObject*': PyObject,
-# 'PyObject**': PyObjectP,
-# 'int*': rffi.INTP}
-
def configure_types():
for config in (CConfig, CConfig2):
for name, TYPE in rffi_platform.configure(config).iteritems():
@@ -952,21 +941,8 @@
wrapper_second_level._dont_inline_ = True
return wrapper_second_level
-def process_va_name(name):
- return name.replace('*', '_star')
-def setup_va_functions(eci):
- for name, TP in VA_TP_LIST.iteritems():
- name_no_star = process_va_name(name)
- func = rffi.llexternal('pypy_va_get_%s' % name_no_star, [VA_LIST_P],
- TP, compilation_info=eci)
- globals()['va_get_%s' % name_no_star] = func
-
-def setup_init_functions(eci, translating):
- if translating:
- prefix = 'PyPy'
- else:
- prefix = 'cpyexttest'
+def setup_init_functions(eci, prefix):
# jump through hoops to avoid releasing the GIL during initialization
# of the cpyext module. The C functions are called with no wrapper,
# but must not do anything like calling back PyType_Ready(). We
@@ -1028,23 +1004,27 @@
# Do not call this more than once per process
def build_bridge(space):
"NOT_RPYTHON"
- from pypy.module.cpyext.pyobject import make_ref
from rpython.translator.c.database import LowLevelDatabase
use_micronumpy = setup_micronumpy(space)
db = LowLevelDatabase()
- prefix ='cpyexttest'
+ prefix = 'cpyexttest'
- functions = generate_decls_and_callbacks(db, prefix=prefix)
+ generate_decls_and_callbacks(db, prefix=prefix)
# Structure declaration code
+ functions = []
members = []
structindex = {}
for header, header_functions in FUNCTIONS_BY_HEADER.iteritems():
for name, func in header_functions.iteritems():
- if not func:
- # added only for the macro, not the decl
- continue
restype, args = c_function_signature(db, func)
+ callargs = ', '.join('arg%d' % (i,)
+ for i in range(len(func.argtypes)))
+ if func.restype is lltype.Void:
+ body = "{ _pypyAPI.%s(%s); }" % (name, callargs)
+ else:
+ body = "{ return _pypyAPI.%s(%s); }" % (name, callargs)
+ functions.append('%s %s(%s)\n%s' % (restype, name, args, body))
members.append('%s (*%s)(%s);' % (restype, name, args))
structindex[name] = len(structindex)
structmembers = '\n'.join(members)
@@ -1056,15 +1036,8 @@
""" % dict(members=structmembers)
global_objects = []
- for name, (typ, expr) in GLOBALS.iteritems():
- if '#' in name:
- continue
- if typ == 'PyDateTime_CAPI*':
- continue
- elif name.startswith('PyExc_'):
- global_objects.append('%s _%s;' % (typ[:-1], name))
- else:
- global_objects.append('%s %s = NULL;' % (typ, name))
+ for name in all_exceptions:
+ global_objects.append('PyTypeObject _PyExc_%s;' % name)
global_code = '\n'.join(global_objects)
prologue = ("#include <Python.h>\n"
@@ -1081,90 +1054,69 @@
'\n' +
'\n'.join(functions))
- eci = build_eci(True, code, use_micronumpy)
+ eci = build_eci(code, use_micronumpy, translating=False)
eci = eci.compile_shared_lib(
outputfilename=str(udir / "module_cache" / "pypyapi"))
+ space.fromcache(State).install_dll(eci)
modulename = py.path.local(eci.libraries[-1])
- def dealloc_trigger():
- from pypy.module.cpyext.pyobject import decref
- print 'dealloc_trigger...'
- while True:
- ob = rawrefcount.next_dead(PyObject)
- if not ob:
- break
- print 'deallocating PyObject', ob
- decref(space, ob)
- print 'dealloc_trigger DONE'
- return "RETRY"
- rawrefcount.init(dealloc_trigger)
-
run_bootstrap_functions(space)
# load the bridge, and init structure
bridge = ctypes.CDLL(str(modulename), mode=ctypes.RTLD_GLOBAL)
- space.fromcache(State).install_dll(eci)
+ # populate static data
+ builder = space.fromcache(State).builder = TestingObjBuilder()
+ for name, (typ, expr) in GLOBALS.iteritems():
+ if '#' in name:
+ name, header = name.split('#')
+ assert typ in ('PyObject*', 'PyTypeObject*', 'PyIntObject*')
+ isptr = False
+ elif name.startswith('PyExc_'):
+ isptr = False
+ elif typ == 'PyDateTime_CAPI*':
+ isptr = True
+ else:
+ raise ValueError("Unknown static data: %s %s" % (typ, name))
- # populate static data
- builder = space.fromcache(StaticObjectBuilder)
- for name, (typ, expr) in GLOBALS.iteritems():
from pypy.module import cpyext # for the eval() below
w_obj = eval(expr)
- if '#' in name:
- name = name.split('#')[0]
- isptr = False
- else:
- isptr = True
- if name.startswith('PyExc_'):
- isptr = False
-
INTERPLEVEL_API[name] = w_obj
- name = name.replace('Py', prefix)
+ mname = mangle_name(prefix, name)
if isptr:
- ptr = ctypes.c_void_p.in_dll(bridge, name)
- if typ == 'PyObject*':
- value = make_ref(space, w_obj)
- elif typ == 'PyDateTime_CAPI*':
- value = w_obj
- else:
- assert False, "Unknown static pointer: %s %s" % (typ, name)
+ assert typ == 'PyDateTime_CAPI*'
+ value = w_obj
+ ptr = ctypes.c_void_p.in_dll(bridge, mname)
ptr.value = ctypes.cast(ll2ctypes.lltype2ctypes(value),
ctypes.c_void_p).value
elif typ in ('PyObject*', 'PyTypeObject*'):
if name.startswith('PyPyExc_') or name.startswith('cpyexttestExc_'):
# we already have the pointer
- in_dll = ll2ctypes.get_ctypes_type(PyObject).in_dll(bridge, name)
+ in_dll = ll2ctypes.get_ctypes_type(PyObject).in_dll(bridge, mname)
py_obj = ll2ctypes.ctypes2lltype(PyObject, in_dll)
else:
# we have a structure, get its address
- in_dll = ll2ctypes.get_ctypes_type(PyObject.TO).in_dll(bridge, name)
+ in_dll = ll2ctypes.get_ctypes_type(PyObject.TO).in_dll(bridge, mname)
py_obj = ll2ctypes.ctypes2lltype(PyObject, ctypes.pointer(in_dll))
builder.prepare(py_obj, w_obj)
- else:
- assert False, "Unknown static object: %s %s" % (typ, name)
- builder.attach_all()
+ builder.attach_all(space)
pypyAPI = ctypes.POINTER(ctypes.c_void_p).in_dll(bridge, 'pypyAPI')
# implement structure initialization code
for header, header_functions in FUNCTIONS_BY_HEADER.iteritems():
for name, func in header_functions.iteritems():
- if name.startswith('cpyext_') or func is None: # XXX hack
- continue
pypyAPI[structindex[name]] = ctypes.cast(
ll2ctypes.lltype2ctypes(func.get_llhelper(space)),
ctypes.c_void_p)
- setup_va_functions(eci)
- setup_init_functions(eci, translating=False)
+ setup_init_functions(eci, prefix)
return modulename.new(ext='')
-class StaticObjectBuilder:
- def __init__(self, space):
- self.space = space
+class StaticObjectBuilder(object):
+ def __init__(self):
self.static_pyobjs = []
self.static_objs_w = []
self.cpyext_type_init = None
@@ -1180,13 +1132,12 @@
self.static_pyobjs.append(py_obj)
self.static_objs_w.append(w_obj)
- def attach_all(self):
+ def attach_all(self, space):
# this is RPython, called once in pypy-c when it imports cpyext
from pypy.module.cpyext.pyobject import get_typedescr, make_ref
from pypy.module.cpyext.typeobject import finish_type_1, finish_type_2
from pypy.module.cpyext.pyobject import track_reference
#
- space = self.space
static_pyobjs = self.get_static_pyobjs()
static_objs_w = self.static_objs_w
for i in range(len(static_objs_w)):
@@ -1207,6 +1158,12 @@
finish_type_1(space, pto)
finish_type_2(space, pto, w_type)
+class TestingObjBuilder(StaticObjectBuilder):
+ """The StaticObjectBuilder used in tests."""
+
+class TranslationObjBuilder(StaticObjectBuilder):
+ """The StaticObjectBuilder used during translation."""
+
def mangle_name(prefix, name):
if name.startswith('Py'):
@@ -1214,23 +1171,26 @@
elif name.startswith('_Py'):
return '_' + prefix + name[3:]
else:
- return None
+ raise ValueError("Error converting '%s'" % name)
-def generate_decls_and_callbacks(db, api_struct=True, prefix=''):
+def write_header(header_name, decls):
+ lines = [
+ '#define Signed long /* xxx temporary fix */',
+ '#define Unsigned unsigned long /* xxx temporary fix */',
+ '',] + decls + [
+ '',
+ '#undef Signed /* xxx temporary fix */',
+ '#undef Unsigned /* xxx temporary fix */',
+ '']
+ decl_h = udir.join(header_name)
+ decl_h.write('\n'.join(lines))
+
+def generate_decls_and_callbacks(db, prefix=''):
"NOT_RPYTHON"
pypy_macros = []
- export_symbols = sorted(FUNCTIONS) + sorted(SYMBOLS_C) + sorted(GLOBALS)
- for name in export_symbols:
- if '#' in name:
- name, header = name.split('#')
- else:
- header = pypy_decl
+ for name in SYMBOLS_C:
newname = mangle_name(prefix, name)
- assert newname, name
- if header == pypy_decl:
- pypy_macros.append('#define %s %s' % (name, newname))
- if name.startswith("PyExc_"):
- pypy_macros.append('#define _%s _%s' % (name, newname))
+ pypy_macros.append('#define %s %s' % (name, newname))
# Generate defines
for macro_name, size in [
@@ -1250,50 +1210,18 @@
pypy_macros_h = udir.join('pypy_macros.h')
pypy_macros_h.write('\n'.join(pypy_macros))
- # implement function callbacks and generate function decls
- functions = []
- decls = {}
- pypy_decls = decls[pypy_decl] = []
- pypy_decls.append('#define Signed long /* xxx temporary fix */\n')
- pypy_decls.append('#define Unsigned unsigned long /* xxx temporary fix */\n')
-
+ # generate function decls
+ decls = defaultdict(list)
for decl in FORWARD_DECLS:
- pypy_decls.append("%s;" % (decl,))
+ decls[pypy_decl].append("%s;" % (decl,))
for header_name, header_functions in FUNCTIONS_BY_HEADER.iteritems():
- if header_name not in decls:
- header = decls[header_name] = []
- header.append('#define Signed long /* xxx temporary fix */\n')
- header.append('#define Unsigned unsigned long /* xxx temporary fix */\n')
- else:
- header = decls[header_name]
-
+ header = decls[header_name]
for name, func in sorted(header_functions.iteritems()):
- if not func:
- continue
- if header == DEFAULT_HEADER:
- _name = name
- else:
- # this name is not included in pypy_macros.h
- _name = mangle_name(prefix, name)
- assert _name is not None, 'error converting %s' % name
- header.append("#define %s %s" % (name, _name))
+ _name = mangle_name(prefix, name)
+ header.append("#define %s %s" % (name, _name))
restype, args = c_function_signature(db, func)
- header.append("PyAPI_FUNC(%s) %s(%s);" % (restype, _name, args))
- if api_struct:
- callargs = ', '.join('arg%d' % (i,)
- for i in range(len(func.argtypes)))
- if func.restype is lltype.Void:
- body = "{ _pypyAPI.%s(%s); }" % (_name, callargs)
- else:
- body = "{ return _pypyAPI.%s(%s); }" % (_name, callargs)
- functions.append('%s %s(%s)\n%s' % (restype, name, args, body))
- for name in VA_TP_LIST:
- name_no_star = process_va_name(name)
- header = ('%s pypy_va_get_%s(va_list* vp)' %
- (name, name_no_star))
- pypy_decls.append('RPY_EXTERN ' + header + ';')
- functions.append(header + '\n{return va_arg(*vp, %s);}\n' % name)
+ header.append("PyAPI_FUNC(%s) %s(%s);" % (restype, name, args))
for name, (typ, expr) in GLOBALS.iteritems():
if '#' in name:
@@ -1302,19 +1230,11 @@
elif name.startswith('PyExc_'):
typ = 'PyObject*'
header = pypy_decl
- if header != pypy_decl:
- decls[header].append('#define %s %s' % (name, mangle_name(prefix, name)))
+ decls[header].append('#define %s %s' % (name, mangle_name(prefix, name)))
decls[header].append('PyAPI_DATA(%s) %s;' % (typ, name))
- for header_name in FUNCTIONS_BY_HEADER.keys():
- header = decls[header_name]
- header.append('#undef Signed /* xxx temporary fix */\n')
- header.append('#undef Unsigned /* xxx temporary fix */\n')
-
for header_name, header_decls in decls.iteritems():
- decl_h = udir.join(header_name)
- decl_h.write('\n'.join(header_decls))
- return functions
+ write_header(header_name, header_decls)
separate_module_files = [source_dir / "varargwrapper.c",
source_dir / "pyerrors.c",
@@ -1335,14 +1255,16 @@
source_dir / "bytesobject.c",
]
-def build_eci(building_bridge, code, use_micronumpy=False):
+def build_eci(code, use_micronumpy=False, translating=False):
"NOT_RPYTHON"
# Build code and get pointer to the structure
kwds = {}
compile_extra=['-DPy_BUILD_CORE']
- if building_bridge:
+ if translating:
+ kwds["includes"] = ['Python.h'] # this is our Python.h
+ else:
if sys.platform == "win32":
# '%s' undefined; assuming extern returning int
compile_extra.append("/we4013")
@@ -1352,8 +1274,6 @@
elif sys.platform.startswith('linux'):
compile_extra.append("-Werror=implicit-function-declaration")
compile_extra.append('-g')
- else:
- kwds["includes"] = ['Python.h'] # this is our Python.h
# Generate definitions for global structures
structs = ["#include <Python.h>"]
@@ -1407,9 +1327,7 @@
return use_micronumpy
# import registers api functions by side-effect, we also need HEADER
from pypy.module.cpyext.ndarrayobject import HEADER
- global FUNCTIONS_BY_HEADER, separate_module_files
- for func_name in ['PyArray_Type', '_PyArray_FILLWBYTE', '_PyArray_ZEROS']:
- FUNCTIONS_BY_HEADER.setdefault(HEADER, {})[func_name] = None
+ global separate_module_files
register_global("PyArray_Type",
'PyTypeObject*', "space.gettypeobject(W_NDimArray.typedef)",
header=HEADER)
@@ -1423,21 +1341,19 @@
db = LowLevelDatabase()
prefix = 'PyPy'
- functions = generate_decls_and_callbacks(db, api_struct=False, prefix=prefix)
+ generate_decls_and_callbacks(db, prefix=prefix)
+
code = "#include <Python.h>\n"
if use_micronumpy:
code += "#include <pypy_numpy.h> /* api.py line 1290 */\n"
- code += "\n".join(functions)
- eci = build_eci(False, code, use_micronumpy)
-
+ eci = build_eci(code, use_micronumpy, translating=True)
space.fromcache(State).install_dll(eci)
run_bootstrap_functions(space)
- setup_va_functions(eci)
# emit uninitialized static data
- builder = space.fromcache(StaticObjectBuilder)
+ builder = space.fromcache(State).builder = TranslationObjBuilder()
lines = ['PyObject *pypy_static_pyobjs[] = {\n']
include_lines = ['RPY_EXTERN PyObject *pypy_static_pyobjs[];\n']
for name, (typ, expr) in sorted(GLOBALS.items()):
@@ -1445,17 +1361,15 @@
name, header = name.split('#')
assert typ in ('PyObject*', 'PyTypeObject*')
typ = typ[:-1]
- if header != pypy_decl:
- # since the #define is not in pypy_macros, do it here
- mname = mangle_name(prefix, name)
- include_lines.append('#define %s %s\n' % (name, mname))
+ mname = mangle_name(prefix, name)
+ include_lines.append('#define %s %s\n' % (name, mname))
elif name.startswith('PyExc_'):
typ = 'PyTypeObject'
name = '_' + name
elif typ == 'PyDateTime_CAPI*':
continue
else:
- assert False, "Unknown static data: %s %s" % (typ, name)
+ raise ValueError("Unknown static data: %s %s" % (typ, name))
from pypy.module import cpyext # for the eval() below
w_obj = eval(expr)
@@ -1475,20 +1389,15 @@
for header, header_functions in FUNCTIONS_BY_HEADER.iteritems():
for name, func in header_functions.iteritems():
- if not func:
- continue
- newname = mangle_name('PyPy', name) or name
+ newname = mangle_name(prefix, name)
deco = entrypoint_lowlevel("cpyext", func.argtypes, newname,
relax=True)
deco(func.get_wrapper(space))
- setup_init_functions(eci, translating=True)
+ setup_init_functions(eci, prefix)
trunk_include = pypydir.dirpath() / 'include'
copy_header_files(trunk_include, use_micronumpy)
-def init_static_data_translated(space):
- builder = space.fromcache(StaticObjectBuilder)
- builder.attach_all()
def _load_from_cffi(space, name, path, initptr):
from pypy.module._cffi_backend import cffi1_module
diff --git a/pypy/module/cpyext/include/complexobject.h b/pypy/module/cpyext/include/complexobject.h
--- a/pypy/module/cpyext/include/complexobject.h
+++ b/pypy/module/cpyext/include/complexobject.h
@@ -16,23 +16,8 @@
Py_complex cval;
} PyComplexObject;
-/* generated function */
-PyAPI_FUNC(int) _PyComplex_AsCComplex(PyObject *, Py_complex *);
-PyAPI_FUNC(PyObject *) _PyComplex_FromCComplex(Py_complex *);
-
-Py_LOCAL_INLINE(Py_complex) PyComplex_AsCComplex(PyObject *obj)
-{
- Py_complex result;
- _PyComplex_AsCComplex(obj, &result);
- return result;
-}
-
-// shmuller 2013/07/30: Make a function, since macro will fail in C++ due to
-// const correctness if called with "const Py_complex"
-//#define PyComplex_FromCComplex(c) _PyComplex_FromCComplex(&c)
-Py_LOCAL_INLINE(PyObject *) PyComplex_FromCComplex(Py_complex c) {
- return _PyComplex_FromCComplex(&c);
-}
+PyAPI_FUNC(Py_complex) PyComplex_AsCComplex(PyObject *obj);
+PyAPI_FUNC(PyObject *) PyComplex_FromCComplex(Py_complex c);
#ifdef __cplusplus
}
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -37,8 +37,24 @@
"""
Fills a newly allocated PyMemoryViewObject with the given W_MemoryView object.
"""
+ assert isinstance(w_obj, W_MemoryView)
py_obj = rffi.cast(PyMemoryViewObject, py_obj)
- py_obj.c_view.c_obj = rffi.cast(PyObject, 0)
+ view = py_obj.c_view
+ ndim = w_obj.buf.getndim()
+ if ndim >= Py_MAX_NDIMS:
+ # XXX warn?
+ return
+ fill_Py_buffer(space, w_obj.buf, view)
+ try:
+ view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
+ view.c_obj = make_ref(space, w_userdata)
+ rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
+ except ValueError:
+ w_s = w_obj.descr_tobytes(space)
+ view.c_obj = make_ref(space, w_s)
+ view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(space.str_w(w_s),
+ track_allocation=False))
+ rffi.setintfield(view, 'c_readonly', 1)
def memory_realize(space, py_obj):
"""
@@ -79,29 +95,13 @@
return generic_cpy_call(space, pb.c_bf_getbuffer, exporter, view, flags)
@cpython_api([PyObject], Py_bufferP, error=CANNOT_FAIL)
-def PyMemoryView_GET_BUFFER(space, w_obj):
+def PyMemoryView_GET_BUFFER(space, pyobj):
"""Return a pointer to the buffer-info structure wrapped by the given
object. The object must be a memoryview instance; this macro doesn't
check its type, you must do it yourself or you will risk crashes."""
- if not isinstance(w_obj, W_MemoryView):
- return lltype.nullptr(Py_buffer)
- py_memobj = rffi.cast(PyMemoryViewObject, as_pyobj(space, w_obj)) # no inc_ref
- view = py_memobj.c_view
- ndim = w_obj.buf.getndim()
- if ndim >= Py_MAX_NDIMS:
- # XXX warn?
- return view
- fill_Py_buffer(space, w_obj.buf, view)
- try:
- view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
- #view.c_obj = make_ref(space, w_obj) # NO - this creates a ref cycle!
- rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
- except ValueError:
- w_s = w_obj.descr_tobytes(space)
- view.c_obj = make_ref(space, w_s)
- view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(space.str_w(w_s), track_allocation=False))
- rffi.setintfield(view, 'c_readonly', 1)
- return view
+ # XXX move to a c-macro
+ py_memobj = rffi.cast(PyMemoryViewObject, pyobj)
+ return py_memobj.c_view
def fill_Py_buffer(space, buf, view):
# c_buf, c_obj have been filled in
@@ -193,9 +193,11 @@
return (_IsCContiguous(view) or _IsFortranContiguous(view))
return 0
- at cpython_api([PyObject], PyObject)
+ at cpython_api([PyObject], PyObject, result_is_ll=True)
def PyMemoryView_FromObject(space, w_obj):
- return space.call_method(space.builtin, "memoryview", w_obj)
+ w_memview = space.call_method(space.builtin, "memoryview", w_obj)
+ py_memview = make_ref(space, w_memview, w_obj)
+ return py_memview
@cpython_api([Py_bufferP], PyObject)
def PyMemoryView_FromBuffer(space, view):
@@ -203,6 +205,7 @@
The memoryview object then owns the buffer represented by view, which
means you shouldn't try to call PyBuffer_Release() yourself: it
will be done on deallocation of the memoryview object."""
+ assert view.c_obj
w_obj = from_ref(space, view.c_obj)
if isinstance(w_obj, W_MemoryView):
return w_obj
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -7,7 +7,7 @@
from rpython.rlib import rgc # Force registration of gc.collect
from pypy.module.cpyext.api import (
cpython_api, generic_cpy_call, PyObject, Py_ssize_t,
- Py_buffer, mangle_name, pypy_decl)
+ pypy_decl, Py_buffer, Py_bufferP)
from pypy.module.cpyext.typeobjectdefs import (
unaryfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc,
getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
@@ -342,7 +342,7 @@
else:
#do not call twice
return
- if self.releasebufferproc:
+ if self.releasebufferproc:
func_target = rffi.cast(releasebufferproc, self.releasebufferproc)
with lltype.scoped_alloc(Py_buffer) as pybuf:
pybuf.c_buf = self.ptr
@@ -407,7 +407,7 @@
size = generic_cpy_call(space, func_target, w_self, index, ptr)
if size < 0:
space.fromcache(State).check_and_raise_exception(always=True)
- buf = CPyBuffer(space, ptr[0], size, w_self,
+ buf = CPyBuffer(space, ptr[0], size, w_self,
releasebuffer=releasebuffer)
fq.register_finalizer(buf)
return space.newbuffer(buf)
@@ -516,7 +516,7 @@
w_type = space.gettypeobject(typedef)
header = pypy_decl
- if mangle_name('', typedef.name) is None:
+ if not (name.startswith('Py') or name.startswith('_Py')):
header = None
handled = False
# unary functions
diff --git a/pypy/module/cpyext/src/complexobject.c b/pypy/module/cpyext/src/complexobject.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/src/complexobject.c
@@ -0,0 +1,16 @@
+
+#include "Python.h"
+
+Py_complex
+PyComplex_AsCComplex(PyObject *obj)
+{
+ Py_complex result;
+ _PyComplex_AsCComplex(obj, &result);
+ return result;
+}
+
+PyObject *
+PyComplex_FromCComplex(Py_complex c)
+{
+ return _PyComplex_FromCComplex(&c);
+}
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -2,7 +2,6 @@
from rpython.rtyper.lltypesystem import rffi, lltype
from pypy.interpreter.error import OperationError, oefmt
from pypy.interpreter import executioncontext
-from rpython.rtyper.lltypesystem import lltype
from rpython.rtyper.annlowlevel import llhelper
from rpython.rlib.rdynload import DLLHANDLE
from rpython.rlib import rawrefcount
@@ -14,9 +13,7 @@
self.reset()
self.programname = lltype.nullptr(rffi.CWCHARP.TO)
self.version = lltype.nullptr(rffi.CCHARP.TO)
- if space.config.translation.gc != "boehm":
- pyobj_dealloc_action = PyObjDeallocAction(space)
- self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
+ self.builder = None
def reset(self):
from pypy.module.cpyext.modsupport import PyMethodDef
@@ -57,22 +54,40 @@
"Function returned an error result without setting an "
"exception")
- def build_api(self, space):
+ def setup_rawrefcount(self):
+ space = self.space
+ if not self.space.config.translating:
+ def dealloc_trigger():
+ from pypy.module.cpyext.pyobject import PyObject, decref
+ print 'dealloc_trigger...'
+ while True:
+ ob = rawrefcount.next_dead(PyObject)
+ if not ob:
+ break
+ print 'deallocating PyObject', ob
+ decref(space, ob)
+ print 'dealloc_trigger DONE'
+ return "RETRY"
+ rawrefcount.init(dealloc_trigger)
+ else:
+ if space.config.translation.gc == "boehm":
+ action = BoehmPyObjDeallocAction(space)
+ space.actionflag.register_periodic_action(action,
+ use_bytecode_counter=True)
+ else:
+ pyobj_dealloc_action = PyObjDeallocAction(space)
+ self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
+
+ def build_api(self):
"""NOT_RPYTHON
This function is called when at object space creation,
and drives the compilation of the cpyext library
"""
from pypy.module.cpyext import api
- state = self.space.fromcache(State)
if not self.space.config.translating:
- state.api_lib = str(api.build_bridge(self.space))
+ self.api_lib = str(api.build_bridge(self.space))
else:
api.setup_library(self.space)
- #
- if self.space.config.translation.gc == "boehm":
- action = BoehmPyObjDeallocAction(self.space)
- self.space.actionflag.register_periodic_action(action,
- use_bytecode_counter=True)
def install_dll(self, eci):
"""NOT_RPYTHON
@@ -84,17 +99,17 @@
def startup(self, space):
"This function is called when the program really starts"
-
from pypy.module.cpyext.typeobject import setup_new_method_def
from pypy.module.cpyext.api import INIT_FUNCTIONS
- from pypy.module.cpyext.api import init_static_data_translated
if we_are_translated():
if space.config.translation.gc != "boehm":
+ # This must be called in RPython, the untranslated version
+ # does something different. Sigh.
rawrefcount.init(
llhelper(rawrefcount.RAWREFCOUNT_DEALLOC_TRIGGER,
self.dealloc_trigger))
- init_static_data_translated(space)
+ self.builder.attach_all(space)
setup_new_method_def(space)
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -89,9 +89,9 @@
def test_typedef(self, space):
from rpython.translator.c.database import LowLevelDatabase
db = LowLevelDatabase()
- assert (api.c_function_signature(db, api.FUNCTIONS['PyPy_TypedefTest1'])
+ assert (api.c_function_signature(db, PyPy_TypedefTest1.api_func)
== ('Py_ssize_t', 'Py_ssize_t arg0'))
- assert (api.c_function_signature(db, api.FUNCTIONS['PyPy_TypedefTest2'])
+ assert (api.c_function_signature(db, PyPy_TypedefTest2.api_func)
== ('Py_ssize_t *', 'Py_ssize_t *arg0'))
PyPy_TypedefTest1(space, 0)
@@ -100,7 +100,7 @@
PyPy_TypedefTest2(space, ppos)
lltype.free(ppos, flavor='raw')
- at pytest.mark.skipif(os.environ.get('USER')=='root',
+ at pytest.mark.skipif(os.environ.get('USER')=='root',
reason='root can write to all files')
def test_copy_header_files(tmpdir):
api.copy_header_files(tmpdir, True)
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -27,8 +27,9 @@
class TestApi:
def test_signature(self):
- assert 'PyModule_Check' in api.FUNCTIONS
- assert api.FUNCTIONS['PyModule_Check'].argtypes == [api.PyObject]
+ common_functions = api.FUNCTIONS_BY_HEADER[api.pypy_decl]
+ assert 'PyModule_Check' in common_functions
+ assert common_functions['PyModule_Check'].argtypes == [api.PyObject]
class SpaceCompiler(SystemCompilationInfo):
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -4,15 +4,15 @@
from pypy.module.cpyext.test.test_api import BaseApiTest
from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
from rpython.rlib.buffer import StringBuffer
+from pypy.module.cpyext.pyobject import from_ref
only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names"
class TestMemoryViewObject(BaseApiTest):
def test_fromobject(self, space, api):
w_hello = space.newbytes("hello")
- # implemented as a C macro
- #assert api.PyObject_CheckBuffer(w_hello)
- w_view = api.PyMemoryView_FromObject(w_hello)
+ assert api.PyObject_CheckBuffer(w_hello)
+ w_view = from_ref(space, api.PyMemoryView_FromObject(w_hello))
w_char = space.call_method(w_view, '__getitem__', space.wrap(0))
assert space.eq_w(w_char, space.wrap('h'))
w_bytes = space.call_method(w_view, "tobytes")
@@ -20,7 +20,7 @@
def test_frombuffer(self, space, api):
w_buf = space.newbuffer(StringBuffer("hello"))
- w_memoryview = api.PyMemoryView_FromObject(w_buf)
+ w_memoryview = from_ref(space, api.PyMemoryView_FromObject(w_buf))
view = api.PyMemoryView_GET_BUFFER(w_memoryview)
assert view.c_ndim == 1
f = rffi.charp2str(view.c_format)
@@ -69,7 +69,9 @@
assert y.format == 'i'
assert y.shape == (10,)
assert len(y) == 10
- assert y[3] == 3
+ s = y[3]
+ assert len(s) == struct.calcsize('i')
+ assert s == struct.pack('i', 3)
def test_buffer_protocol_capi(self):
foo = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -86,7 +86,7 @@
py_getsetdef.c_set = rffi.cast(setter, 0)
py_getsetdef.c_closure = rffi.cast(rffi.VOIDP, 0)
return py_getsetdef
-
+
class W_MemberDescr(GetSetProperty):
name = 'member_descriptor'
@@ -619,7 +619,7 @@
pto.c_tp_free = llslot(space, PyObject_Free)
pto.c_tp_alloc = llslot(space, PyType_GenericAlloc)
- builder = space.fromcache(StaticObjectBuilder)
+ builder = space.fromcache(State).builder
if ((pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE) != 0
and builder.cpyext_type_init is None):
# this ^^^ is not None only during startup of cpyext. At that
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -541,6 +541,10 @@
sub = self._op_val(space, w_old)
by = self._op_val(space, w_new)
+ # the following two lines are for being bug-to-bug compatible
+ # with CPython: see issue #2448
+ if count >= 0 and len(input) == 0:
+ return self._empty()
try:
res = replace(input, sub, by, count)
except OverflowError:
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -874,6 +874,16 @@
s = b"a" * (2**16)
raises(OverflowError, s.replace, b"", s)
+ def test_replace_issue2448(self):
+ # CPython's replace() method has a bug that makes
+ # ''.replace('', 'x') gives a different answer than
+ # ''.replace('', 'x', 1000). This is the case in all
+ # known versions, at least until 2.7.13. Some people
+ # call that a feature on the CPython issue report and
+ # the discussion dies out, so it might never be fixed.
+ assert ''.replace('', 'x') == 'x'
+ assert ''.replace('', 'x', 1000) == ''
+
def test_getslice(self):
s = b"abc"
assert s[:] == b"abc"
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -185,9 +185,10 @@
assert m[2] == 1
def test_pypy_raw_address_base(self):
- raises(ValueError, memoryview(b"foobar")._pypy_raw_address)
- a = memoryview(bytearray(b"foobar"))._pypy_raw_address()
+ a = memoryview("foobar")._pypy_raw_address()
assert a != 0
+ b = memoryview(bytearray("foobar"))._pypy_raw_address()
+ assert b != 0
def test_hex(self):
assert memoryview(b"abc").hex() == u'616263'
diff --git a/rpython/jit/codewriter/codewriter.py b/rpython/jit/codewriter/codewriter.py
--- a/rpython/jit/codewriter/codewriter.py
+++ b/rpython/jit/codewriter/codewriter.py
@@ -106,8 +106,9 @@
else:
name = 'unnamed' % id(ssarepr)
i = 1
- # escape <lambda> names for windows
- name = name.replace('<lambda>', '_(lambda)_')
+ # escape names like <lambda> for windows by removing any strange
+ # character; then make sure the names are not too long
+ name = ''.join(c for c in name if c.isalnum() or c == '_')[:60]
extra = ''
while dir.join(name+extra).check():
i += 1
diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -210,7 +210,6 @@
return rlist.ll_pop(rlist.dum_checkidx, l, index)
_ll_2_list_append = rlist.ll_append
_ll_2_list_extend = rlist.ll_extend
-_ll_3_list_insert = rlist.ll_insert_nonneg
_ll_2_list_delslice_startonly = rlist.ll_listdelslice_startonly
_ll_3_list_delslice_startstop = rlist.ll_listdelslice_startstop
_ll_2_list_inplace_mul = rlist.ll_inplace_mul
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4613,3 +4613,10 @@
self.check_operations_history(guard_nonnull=0, guard_nonnull_class=0,
guard_class=2,
assert_not_none=2) # before optimization
+
+ def test_call_time_clock(self):
+ import time
+ def g():
+ time.clock()
+ return 0
+ self.interp_operations(g, [])
diff --git a/rpython/jit/metainterp/test/test_list.py b/rpython/jit/metainterp/test/test_list.py
--- a/rpython/jit/metainterp/test/test_list.py
+++ b/rpython/jit/metainterp/test/test_list.py
@@ -212,6 +212,8 @@
s += lst[0]
lst.pop()
lst.append(1)
+ lst.insert(0, 5)
+ lst.insert(1, 6)
s *= lst.pop()
return s
res = self.meta_interp(f, [15], listops=True)
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -158,7 +158,11 @@
# record that ignore_finalizer() has been called
GCFLAG_IGNORE_FINALIZER = first_gcflag << 10
-_GCFLAG_FIRST_UNUSED = first_gcflag << 11 # the first unused bit
+# shadow objects can have its memory initialized when it is created.
+# It does not need an additional copy in trace out
+GCFLAG_SHADOW_INITIALIZED = first_gcflag << 11
+
+_GCFLAG_FIRST_UNUSED = first_gcflag << 12 # the first unused bit
# States for the incremental GC
@@ -729,6 +733,16 @@
obj = self.external_malloc(typeid, length, alloc_young=True)
return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
+ def move_out_of_nursery(self, obj):
+ # called twice, it should return the same shadow object,
+ # and not creating another shadow object
+ if self.header(obj).tid & GCFLAG_HAS_SHADOW:
+ shadow = self.nursery_objects_shadows.get(obj)
+ ll_assert(shadow != llmemory.NULL,
+ "GCFLAG_HAS_SHADOW but no shadow found")
+ return shadow
+
+ return self._allocate_shadow(obj, copy=True)
def collect(self, gen=2):
"""Do a minor (gen=0), start a major (gen=1), or do a full
@@ -1756,11 +1770,11 @@
#
# clear the arena between the last pinned object (or arena start)
# and the pinned object
- pinned_obj_size = llarena.getfakearenaaddress(cur) - prev
+ free_range_size = llarena.getfakearenaaddress(cur) - prev
if self.gc_nursery_debug:
- llarena.arena_reset(prev, pinned_obj_size, 3)
+ llarena.arena_reset(prev, free_range_size, 3)
else:
- llarena.arena_reset(prev, pinned_obj_size, 0)
+ llarena.arena_reset(prev, free_range_size, 0)
#
# clean up object's flags
obj = cur + size_gc_header
@@ -1770,7 +1784,7 @@
nursery_barriers.append(cur)
#
# update 'prev' to the end of the 'cur' object
- prev = prev + pinned_obj_size + \
+ prev = prev + free_range_size + \
(size_gc_header + self.get_size(obj))
#
# reset everything after the last pinned object till the end of the arena
@@ -1982,6 +1996,9 @@
and self.young_rawmalloced_objects.contains(obj)):
self._visit_young_rawmalloced_object(obj)
return
+ # copy the contents of the object? usually yes, but not for some
+ # shadow objects
+ copy = True
#
size_gc_header = self.gcheaderbuilder.size_gc_header
if self.header(obj).tid & (GCFLAG_HAS_SHADOW | GCFLAG_PINNED) == 0:
@@ -2037,13 +2054,18 @@
# Remove the flag GCFLAG_HAS_SHADOW, so that it doesn't get
# copied to the shadow itself.
self.header(obj).tid &= ~GCFLAG_HAS_SHADOW
+ tid = self.header(obj).tid
+ if (tid & GCFLAG_SHADOW_INITIALIZED) != 0:
+ copy = False
+ self.header(obj).tid &= ~GCFLAG_SHADOW_INITIALIZED
#
totalsize = size_gc_header + self.get_size(obj)
self.nursery_surviving_size += raw_malloc_usage(totalsize)
#
# Copy it. Note that references to other objects in the
# nursery are kept unchanged in this step.
- llmemory.raw_memcopy(obj - size_gc_header, newhdr, totalsize)
+ if copy:
+ llmemory.raw_memcopy(obj - size_gc_header, newhdr, totalsize)
#
# Set the old object's tid to -42 (containing all flags) and
# replace the old object's content with the target address.
@@ -2570,7 +2592,8 @@
# ----------
# id() and identityhash() support
- def _allocate_shadow(self, obj):
+ @specialize.arg(2)
+ def _allocate_shadow(self, obj, copy=False):
size_gc_header = self.gcheaderbuilder.size_gc_header
size = self.get_size(obj)
shadowhdr = self._malloc_out_of_nursery(size_gc_header +
@@ -2592,6 +2615,12 @@
#
self.header(obj).tid |= GCFLAG_HAS_SHADOW
self.nursery_objects_shadows.setitem(obj, shadow)
+
+ if copy:
+ self.header(obj).tid |= GCFLAG_SHADOW_INITIALIZED
+ totalsize = size_gc_header + self.get_size(obj)
+ llmemory.raw_memcopy(obj - size_gc_header, shadow, totalsize)
+
return shadow
def _find_shadow(self, obj):
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -551,6 +551,13 @@
[s_gc, SomeAddress()],
annmodel.s_None)
+ self.move_out_of_nursery_ptr = None
+ if hasattr(GCClass, 'move_out_of_nursery'):
+ self.move_out_of_nursery_ptr = getfn(GCClass.move_out_of_nursery,
+ [s_gc, SomeAddress()],
+ SomeAddress())
+
+
def create_custom_trace_funcs(self, gc, rtyper):
custom_trace_funcs = tuple(rtyper.custom_trace_funcs)
rtyper.custom_trace_funcs = custom_trace_funcs
@@ -1585,6 +1592,17 @@
hop.genop("direct_call", [self.ignore_finalizer_ptr,
self.c_const_gc, v_adr])
+ def gct_gc_move_out_of_nursery(self, hop):
+ if self.move_out_of_nursery_ptr is not None:
+ v_adr = hop.genop("cast_ptr_to_adr", [hop.spaceop.args[0]],
+ resulttype=llmemory.Address)
+ v_ret = hop.genop("direct_call", [self.move_out_of_nursery_ptr,
+ self.c_const_gc, v_adr],
+ resulttype=llmemory.Address)
+ hop.genop("cast_adr_to_ptr", [v_ret],
+ resultvar = hop.spaceop.result)
+
+
class TransformerLayoutBuilder(gctypelayout.TypeLayoutBuilder):
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -2,6 +2,8 @@
Buffer protocol support.
"""
from rpython.rlib import jit
+from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
+ nonmoving_raw_ptr_for_resizable_list)
class Buffer(object):
@@ -84,7 +86,7 @@
def __init__(self, value):
self.value = value
- self.readonly = True
+ self.readonly = 1
def getlength(self):
return len(self.value)
@@ -108,6 +110,9 @@
return self.value[start:stop]
return Buffer.getslice(self, start, stop, step, size)
+ def get_raw_address(self):
+ from rpython.rtyper.lltypesystem import rffi
+ return rffi.get_raw_address_of_string(self.value)
class SubBuffer(Buffer):
_attrs_ = ['buffer', 'offset', 'size', 'readonly']
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -535,6 +535,25 @@
from rpython.rtyper.lltypesystem.lloperation import llop
llop.gc_ignore_finalizer(lltype.Void, obj)
+ at jit.dont_look_inside
+def move_out_of_nursery(obj):
+ """ Returns another object which is a copy of obj; but at any point
+ (either now or in the future) the returned object might suddenly
+ become identical to the one returned.
+
+ NOTE: Only use for immutable objects!
+ """
+ pass
+
+class MoveOutOfNurseryEntry(ExtRegistryEntry):
+ _about_ = move_out_of_nursery
+
+ def compute_result_annotation(self, s_obj):
+ return s_obj
+
+ def specialize_call(self, hop):
+ hop.exception_cannot_occur()
+ return hop.genop('gc_move_out_of_nursery', hop.args_v, resulttype=hop.r_result)
# ____________________________________________________________
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -997,12 +997,12 @@
if signal_checker is not None:
signal_checker()
- def sendto(self, data, flags, address):
+ def sendto(self, data, length, flags, address):
"""Like send(data, flags) but allows specifying the destination
address. (Note that 'flags' is mandatory here.)"""
self.wait_for_data(True)
addr = address.lock()
- res = _c.sendto(self.fd, data, len(data), flags,
+ res = _c.sendto(self.fd, data, length, flags,
addr, address.addrlen)
address.unlock()
if res < 0:
diff --git a/rpython/rlib/test/test_buffer.py b/rpython/rlib/test/test_buffer.py
--- a/rpython/rlib/test/test_buffer.py
+++ b/rpython/rlib/test/test_buffer.py
@@ -1,4 +1,4 @@
-from rpython.rlib.buffer import *
+from rpython.rlib.buffer import StringBuffer, SubBuffer, Buffer
from rpython.annotator.annrpython import RPythonAnnotator
from rpython.annotator.model import SomeInteger
@@ -64,3 +64,10 @@
for i in range(9999, 9, -1):
buf = SubBuffer(buf, 1, i)
assert buf.getlength() == 10
+
+def test_string_buffer_as_buffer():
+ buf = StringBuffer(b'hello world')
+ addr = buf.get_raw_address()
+ assert addr[0] == b'h'
+ assert addr[4] == b'o'
+ assert addr[6] == b'w'
diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py
--- a/rpython/rlib/test/test_rsocket.py
+++ b/rpython/rlib/test/test_rsocket.py
@@ -320,7 +320,7 @@
s2.bind(INETAddress('127.0.0.1', INADDR_ANY))
addr2 = s2.getsockname()
- s1.sendto('?', 0, addr2)
+ s1.sendto('?', 1, 0, addr2)
buf = s2.recv(100)
assert buf == '?'
s2.connect(addr)
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -1138,6 +1138,9 @@
exc_data.exc_value = lltype.typeOf(evalue)._defl()
return bool(etype)
+ def op_gc_move_out_of_nursery(self, obj):
+ raise NotImplementedError("gc_move_out_of_nursery")
+
class Tracer(object):
Counter = 0
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -496,6 +496,8 @@
'gc_rawrefcount_to_obj': LLOp(sideeffects=False),
'gc_rawrefcount_next_dead': LLOp(),
+ 'gc_move_out_of_nursery': LLOp(),
+
# ------- JIT & GC interaction, only for some GCs ----------
'gc_adr_of_nursery_free' : LLOp(),
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -739,6 +739,9 @@
def op_gc_ignore_finalizer(obj):
pass
+def op_gc_move_out_of_nursery(obj):
+ return obj
+
# ____________________________________________________________
def get_op_impl(opname):
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -754,6 +754,7 @@
SIGNED = lltype.Signed
SIGNEDP = lltype.Ptr(lltype.Array(SIGNED, hints={'nolength': True}))
+
# various type mapping
# conversions between str and char*
@@ -876,6 +877,7 @@
get_nonmovingbuffer._always_inline_ = 'try' # get rid of the returned tuple
get_nonmovingbuffer._annenforceargs_ = [strtype]
+
@jit.dont_look_inside
def get_nonmovingbuffer_final_null(data):
tup = get_nonmovingbuffer(data)
@@ -1310,3 +1312,52 @@
releasegil=False,
calling_conv='c',
)
+
+
+if not we_are_translated():
+ class RawBytes(object):
+ # literal copy of _cffi_backend/func.py
+ def __init__(self, string):
+ self.ptr = str2charp(string, track_allocation=False)
+ def __del__(self):
+ if free_charp is not None: # CPython shutdown
+ free_charp(self.ptr, track_allocation=False)
+
+ TEST_RAW_ADDR_KEEP_ALIVE = {}
+
+ at jit.dont_look_inside
+def get_raw_address_of_string(string):
+ """Returns a 'char *' that is valid as long as the rpython string object is alive.
+ Two calls to to this function, given the same string parameter,
+ are guaranteed to return the same pointer.
+
+ The extra parameter key is necessary to create a weak reference.
+ The buffer of the returned pointer (if object is young) lives as long
+ as key is alive. If key goes out of scope, the buffer will eventually
+ be freed. `string` cannot go out of scope until the RawBytes object
+ referencing it goes out of scope.
+ """
+ assert isinstance(string, str)
+ from rpython.rtyper.annlowlevel import llstr
+ from rpython.rtyper.lltypesystem.rstr import STR
+ from rpython.rtyper.lltypesystem import llmemory
+ from rpython.rlib import rgc
+
+ if we_are_translated():
+ if rgc.can_move(string):
+ string = rgc.move_out_of_nursery(string)
+
+ # string cannot move now! return the address
+ lldata = llstr(string)
+ data_start = (llmemory.cast_ptr_to_adr(lldata) +
+ offsetof(STR, 'chars') +
+ llmemory.itemoffsetof(STR.chars, 0))
+ data_start = cast(CCHARP, data_start)
+ data_start[len(string)] = '\x00' # write the final extra null
+ return data_start
+ else:
+ global TEST_RAW_ADDR_KEEP_ALIVE
+ if string in TEST_RAW_ADDR_KEEP_ALIVE:
+ return TEST_RAW_ADDR_KEEP_ALIVE[string].ptr
+ TEST_RAW_ADDR_KEEP_ALIVE[string] = rb = RawBytes(string)
+ return rb.ptr
diff --git a/rpython/rtyper/lltypesystem/test/test_ztranslated.py b/rpython/rtyper/lltypesystem/test/test_ztranslated.py
new file mode 100644
--- /dev/null
+++ b/rpython/rtyper/lltypesystem/test/test_ztranslated.py
@@ -0,0 +1,59 @@
+import gc
+from rpython.translator.c.test.test_genc import compile
+from rpython.rtyper.lltypesystem import rffi
+from rpython.rtyper.lltypesystem import lltype
+from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rlib import rgc
+
+def debug_assert(boolresult, msg):
+ if not boolresult:
+ llop.debug_print(lltype.Void, "\n\nassert failed: %s\n\n" % msg)
+ assert boolresult
+
+def use_str():
+ mystr = b'abc'
+ #debug_assert(rgc.can_move(mystr), "short string cannot move... why?")
+ ptr = rffi.get_raw_address_of_string(mystr)
+ ptr2 = rffi.get_raw_address_of_string(mystr)
+ debug_assert(ptr == ptr2, "ptr != ptr2")
+ debug_assert(ptr[0] == b'a', "notnurseryadr[0] == b'a' is is %s" % ptr[0])
+ ptr[0] = b'x' # oh no no, in real programs nobody is allowed to modify that
+ debug_assert(mystr[0] == b'a', "mystr[0] != b'a'")
+ debug_assert(ptr[0] == b'x', "notnurseryadr[0] == b'x'")
+ gc.collect()
+ nptr = rffi.get_raw_address_of_string(mystr)
+ debug_assert(nptr == ptr, "second call to mystr must return the same ptr")
+ debug_assert(ptr[0] == b'x', "failure a")
+ debug_assert(nptr[0] == b'x', "failure b")
+ mystr = None
+
+def long_str(lstr):
+ ptr = rffi.get_raw_address_of_string(lstr)
+ for i,c in enumerate(lstr):
+ debug_assert(ptr[i] == c, "failure c")
+ gc.collect()
+ ptr2 = rffi.get_raw_address_of_string(lstr)
+ debug_assert(ptr == ptr2, "ptr != ptr2!!!")
+ return ptr
+
+def main(argv=[]):
+ use_str()
+ gc.collect()
+ mystr = b"12341234aa"*4096*10
+ #debug_assert(not rgc.can_move(mystr), "long string can move... why?")
+ p1 = long_str(mystr)
+ gc.collect()
+ copystr = mystr[:]
+ copystr += 'a'
+ p2 = long_str(copystr)
+ debug_assert(p1 != p2, "p1 == p2")
+ return 0
+
+# ____________________________________________________________
+
+def target(driver, args):
+ return main
+
+def test_compiled_incminimark():
+ fn = compile(main, [], gcpolicy="incminimark")
+ fn()
diff --git a/rpython/rtyper/rlist.py b/rpython/rtyper/rlist.py
--- a/rpython/rtyper/rlist.py
+++ b/rpython/rtyper/rlist.py
@@ -588,6 +588,7 @@
l.ll_setitem_fast(length, newitem)
# this one is for the special case of insert(0, x)
+ at jit.look_inside_iff(lambda l,n: jit.isvirtual(l))
def ll_prepend(l, newitem):
length = l.ll_length()
l._ll_resize_ge(length+1) # see "a note about overflows" above
@@ -597,7 +598,6 @@
l.ll_setitem_fast(dst, l.ll_getitem_fast(src))
dst = src
l.ll_setitem_fast(0, newitem)
-ll_prepend.oopspec = 'list.insert(l, 0, newitem)'
def ll_concat(RESLIST, l1, l2):
len1 = l1.ll_length()
@@ -612,6 +612,7 @@
return l
# no oopspec -- the function is inlined by the JIT
+ at jit.look_inside_iff(lambda l,i,n: jit.isvirtual(l) and jit.isconstant(i))
def ll_insert_nonneg(l, index, newitem):
length = l.ll_length()
ll_assert(0 <= index, "negative list insertion index")
@@ -623,7 +624,6 @@
l.ll_setitem_fast(dst, l.ll_getitem_fast(src))
dst = src
l.ll_setitem_fast(index, newitem)
-ll_insert_nonneg.oopspec = 'list.insert(l, index, newitem)'
def ll_pop_nonneg(func, l, index):
ll_assert(index >= 0, "unexpectedly negative list pop index")
diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py
--- a/rpython/rtyper/rstr.py
+++ b/rpython/rtyper/rstr.py
@@ -33,6 +33,10 @@
value, len(value), 'strict', final=True,
errorhandler=self.ll_raise_unicode_exception_decode,
allow_surrogates=False, result=result)
+ # XXX should it really be 'allow_surrogates=False'? In RPython,
+ # unicode.decode('utf-8') happily accepts surrogates. This
+ # makes it hard to test untranslated (it's the cause of a
+ # failure in lib-python's test_warnings on PyPy3, for example)
return self.ll.llunicode(result.build())
@staticmethod
More information about the pypy-commit
mailing list