[pypy-commit] pypy cpyext-realloc: merge default into branch
mattip
pypy.commits at gmail.com
Sun Aug 7 16:24:35 EDT 2016
Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-realloc
Changeset: r86070:a6addb94dc29
Date: 2016-08-07 23:15 +0300
http://bitbucket.org/pypy/pypy/changeset/a6addb94dc29/
Log: merge default into branch
diff too long, truncating to 2000 out of 2979 lines
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: cffi
-Version: 1.7.0
+Version: 1.8.0
Summary: Foreign Function Interface for Python calling C code.
Home-page: http://cffi.readthedocs.org
Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
from .api import FFI, CDefError, FFIError
from .ffiplatform import VerificationError, VerificationMissing
-__version__ = "1.7.0"
-__version_info__ = (1, 7, 0)
+__version__ = "1.8.0"
+__version_info__ = (1, 8, 0)
# The verifier module file names are based on the CRC32 of a string that
# contains the following version number. It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -42,7 +42,9 @@
# include <stdint.h>
# endif
# if _MSC_VER < 1800 /* MSVC < 2013 */
- typedef unsigned char _Bool;
+# ifndef __cplusplus
+ typedef unsigned char _Bool;
+# endif
# endif
#else
# include <stdint.h>
@@ -59,7 +61,7 @@
#ifdef __cplusplus
# ifndef _Bool
-# define _Bool bool /* semi-hackish: C++ has no _Bool; bool is builtin */
+ typedef bool _Bool; /* semi-hackish: C++ has no _Bool; bool is builtin */
# endif
#endif
@@ -196,20 +198,6 @@
return NULL;
}
-_CFFI_UNUSED_FN
-static PyObject **_cffi_unpack_args(PyObject *args_tuple, Py_ssize_t expected,
- const char *fnname)
-{
- if (PyTuple_GET_SIZE(args_tuple) != expected) {
- PyErr_Format(PyExc_TypeError,
- "%.150s() takes exactly %zd arguments (%zd given)",
- fnname, expected, PyTuple_GET_SIZE(args_tuple));
- return NULL;
- }
- return &PyTuple_GET_ITEM(args_tuple, 0); /* pointer to the first item,
- the others follow */
-}
-
/********** end CPython-specific section **********/
#else
_CFFI_UNUSED_FN
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
f = PySys_GetObject((char *)"stderr");
if (f != NULL && f != Py_None) {
PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
- "\ncompiled with cffi version: 1.7.0"
+ "\ncompiled with cffi version: 1.8.0"
"\n_cffi_backend module: ", f);
modules = PyImport_GetModuleDict();
mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -519,12 +519,10 @@
smallest_value = min(self.enumvalues)
largest_value = max(self.enumvalues)
else:
- import warnings
- warnings.warn("%r has no values explicitly defined; next version "
- "will refuse to guess which integer type it is "
- "meant to be (unsigned/signed, int/long)"
- % self._get_c_name())
- smallest_value = largest_value = 0
+ raise api.CDefError("%r has no values explicitly defined: "
+ "refusing to guess which integer type it is "
+ "meant to be (unsigned/signed, int/long)"
+ % self._get_c_name())
if smallest_value < 0: # needs a signed type
sign = 1
candidate1 = PrimitiveType("int")
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -275,6 +275,8 @@
def write_c_source_to_f(self, f, preamble):
self._f = f
prnt = self._prnt
+ if self.ffi._embedding is None:
+ prnt('#define Py_LIMITED_API')
#
# first the '#include' (actually done by inlining the file's content)
lines = self._rel_readlines('_cffi_include.h')
@@ -683,13 +685,11 @@
rng = range(len(tp.args))
for i in rng:
prnt(' PyObject *arg%d;' % i)
- prnt(' PyObject **aa;')
prnt()
- prnt(' aa = _cffi_unpack_args(args, %d, "%s");' % (len(rng), name))
- prnt(' if (aa == NULL)')
+ prnt(' if (!PyArg_UnpackTuple(args, "%s", %d, %d, %s))' % (
+ name, len(rng), len(rng),
+ ', '.join(['&arg%d' % i for i in rng])))
prnt(' return NULL;')
- for i in rng:
- prnt(' arg%d = aa[%d];' % (i, i))
prnt()
#
for i, type in enumerate(tp.args):
@@ -862,6 +862,8 @@
enumfields = list(tp.enumfields())
for fldname, fldtype, fbitsize, fqual in enumfields:
fldtype = self._field_type(tp, fldname, fldtype)
+ self._check_not_opaque(fldtype,
+ "field '%s.%s'" % (tp.name, fldname))
# cname is None for _add_missing_struct_unions() only
op = OP_NOOP
if fbitsize >= 0:
@@ -911,6 +913,13 @@
first_field_index, c_fields))
self._seen_struct_unions.add(tp)
+ def _check_not_opaque(self, tp, location):
+ while isinstance(tp, model.ArrayType):
+ tp = tp.item
+ if isinstance(tp, model.StructOrUnion) and tp.fldtypes is None:
+ raise TypeError(
+ "%s is of an opaque type (not declared in cdef())" % location)
+
def _add_missing_struct_unions(self):
# not very nice, but some struct declarations might be missing
# because they don't have any known C name. Check that they are
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -99,17 +99,24 @@
The garbage collectors used or implemented by PyPy are not based on
reference counting, so the objects are not freed instantly when they are no
-longer reachable. The most obvious effect of this is that files are not
+longer reachable. The most obvious effect of this is that files (and sockets, etc) are not
promptly closed when they go out of scope. For files that are opened for
writing, data can be left sitting in their output buffers for a while, making
the on-disk file appear empty or truncated. Moreover, you might reach your
OS's limit on the number of concurrently opened files.
-Fixing this is essentially impossible without forcing a
+If you are debugging a case where a file in your program is not closed
+properly, you can use the ``-X track-resources`` command line option. If it is
+given, a ``ResourceWarning`` is produced for every file and socket that the
+garbage collector closes. The warning will contain the stack trace of the
+position where the file or socket was created, to make it easier to see which
+parts of the program don't close files explicitly.
+
+Fixing this difference to CPython is essentially impossible without forcing a
reference-counting approach to garbage collection. The effect that you
get in CPython has clearly been described as a side-effect of the
implementation and not a language design decision: programs relying on
-this are basically bogus. It would anyway be insane to try to enforce
+this are basically bogus. It would be a too strong restriction to try to enforce
CPython's behavior in a language spec, given that it has no chance to be
adopted by Jython or IronPython (or any other port of Python to Java or
.NET).
@@ -134,7 +141,7 @@
Here are some more technical details. This issue affects the precise
time at which ``__del__`` methods are called, which
-is not reliable in PyPy (nor Jython nor IronPython). It also means that
+is not reliable or timely in PyPy (nor Jython nor IronPython). It also means that
**weak references** may stay alive for a bit longer than expected. This
makes "weak proxies" (as returned by ``weakref.proxy()``) somewhat less
useful: they will appear to stay alive for a bit longer in PyPy, and
diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst
--- a/pypy/doc/gc_info.rst
+++ b/pypy/doc/gc_info.rst
@@ -14,10 +14,9 @@
Defaults to 1/2 of your cache or ``4M``.
Small values (like 1 or 1KB) are useful for debugging.
-``PYPY_GC_NURSERY_CLEANUP``
- The interval at which nursery is cleaned up. Must
- be smaller than the nursery size and bigger than the
- biggest object we can allotate in the nursery.
+``PYPY_GC_NURSERY_DEBUG``
+ If set to non-zero, will fill nursery with garbage, to help
+ debugging.
``PYPY_GC_INCREMENT_STEP``
The size of memory marked during the marking step. Default is size of
@@ -62,3 +61,8 @@
use.
Values are ``0`` (off), ``1`` (on major collections) or ``2`` (also
on minor collections).
+
+``PYPY_GC_MAX_PINNED``
+ The maximal number of pinned objects at any point in time. Defaults
+ to a conservative value depending on nursery size and maximum object
+ size inside the nursery. Useful for debugging by setting it to 0.
diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst
--- a/pypy/doc/man/pypy.1.rst
+++ b/pypy/doc/man/pypy.1.rst
@@ -2,6 +2,9 @@
pypy
======
+.. note: this is turned into a regular man page "pypy.1" by
+ doing "make man" in pypy/doc/
+
SYNOPSIS
========
@@ -48,6 +51,10 @@
-B
Disable writing bytecode (``.pyc``) files.
+-X track-resources
+ Produce a ``ResourceWarning`` whenever a file or socket is closed by the
+ garbage collector.
+
--version
Print the PyPy version.
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -109,3 +109,18 @@
.. branch: jitlog-exact-source-lines
Log exact line positions in debug merge points.
+
+.. branch: null_byte_after_str
+
+Allocate all RPython strings with one extra byte, normally unused.
+It is used to hold a final zero in case we need some ``char *``
+representation of the string, together with checks like ``not
+can_move()`` or object pinning. Main new thing that this allows:
+``ffi.from_buffer(string)`` in CFFI. Additionally, and most
+importantly, CFFI calls that take directly a string as argument don't
+copy the string any more---this is like CFFI on CPython.
+
+.. branch: resource_warning
+
+Add a new command line option -X track-resources which will produce
+ResourceWarnings when the GC closes unclosed files and sockets.
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -24,11 +24,15 @@
-V : print the Python version number and exit (also --version)
-W arg : warning control; arg is action:message:category:module:lineno
also PYTHONWARNINGS=arg
+-X arg : set implementation-specific option
file : program read from script file
- : program read from stdin (default; interactive mode if a tty)
arg ...: arguments passed to program in sys.argv[1:]
+
PyPy options and arguments:
--info : print translation information about this PyPy executable
+-X track-resources : track the creation of files and sockets and display
+ a warning if they are not closed explicitly
"""
# Missing vs CPython: PYTHONHOME, PYTHONCASEOK
USAGE2 = """
@@ -229,6 +233,14 @@
import pypyjit
pypyjit.set_param(jitparam)
+def set_runtime_options(options, Xparam, *args):
+ if Xparam == 'track-resources':
+ sys.pypy_set_track_resources(True)
+ else:
+ print >> sys.stderr, 'usage: %s -X [options]' % (get_sys_executable(),)
+ print >> sys.stderr, '[options] can be: track-resources'
+ raise SystemExit
+
class CommandLineError(Exception):
pass
@@ -404,6 +416,7 @@
'--info': (print_info, None),
'--jit': (set_jit_option, Ellipsis),
'-funroll-loops': (funroll_loops, None),
+ '-X': (set_runtime_options, Ellipsis),
'--': (end_options, None),
}
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -108,8 +108,15 @@
return getattr(space, name)(operand)
return do_fold
-def _fold_pow(space, left, right):
- return space.pow(left, right, space.w_None)
+def _fold_pow(space, w_left, w_right):
+ # don't constant-fold if "w_left" and "w_right" are integers and
+ # the estimated bit length of the power is unreasonably large
+ space.appexec([w_left, w_right], """(left, right):
+ if isinstance(left, (int, long)) and isinstance(right, (int, long)):
+ if left.bit_length() * right > 5000:
+ raise OverflowError
+ """)
+ return space.pow(w_left, w_right, space.w_None)
def _fold_not(space, operand):
return space.wrap(not space.is_true(operand))
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1156,3 +1156,22 @@
counts = self.count_instructions(source)
assert ops.BUILD_SET not in counts
assert ops.LOAD_CONST in counts
+
+ def test_dont_fold_huge_powers(self):
+ for source in (
+ "2 ** 3000", # not constant-folded: too big
+ "(-2) ** 3000",
+ ):
+ source = 'def f(): %s' % source
+ counts = self.count_instructions(source)
+ assert ops.BINARY_POWER in counts
+
+ for source in (
+ "2 ** 2000", # constant-folded
+ "2 ** -3000",
+ "1.001 ** 3000",
+ "1 ** 3000.0",
+ ):
+ source = 'def f(): %s' % source
+ counts = self.count_instructions(source)
+ assert ops.BINARY_POWER not in counts
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1764,6 +1764,40 @@
_warnings.warn(msg, warningcls, stacklevel=stacklevel)
""")
+ def resource_warning(self, w_msg, w_tb):
+ self.appexec([w_msg, w_tb],
+ """(msg, tb):
+ import sys
+ print >> sys.stderr, msg
+ if tb:
+ print >> sys.stderr, "Created at (most recent call last):"
+ print >> sys.stderr, tb
+ """)
+
+ def format_traceback(self):
+ # we need to disable track_resources before calling the traceback
+ # module. Else, it tries to open more files to format the traceback,
+ # the file constructor will call space.format_traceback etc., in an
+ # inifite recursion
+ flag = self.sys.track_resources
+ self.sys.track_resources = False
+ try:
+ return self.appexec([],
+ """():
+ import sys, traceback
+ # the "1" is because we don't want to show THIS code
+ # object in the traceback
+ try:
+ f = sys._getframe(1)
+ except ValueError:
+ # this happens if you call format_traceback at the very beginning
+ # of startup, when there is no bottom code object
+ return '<no stacktrace available>'
+ return "".join(traceback.format_stack(f))
+ """)
+ finally:
+ self.sys.track_resources = flag
+
class AppExecCache(SpaceCache):
def build(cache, source):
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -220,6 +220,13 @@
expected = {"no_user_site": True}
self.check(['-c', 'pass'], {}, sys_argv=['-c'], run_command='pass', **expected)
+ def test_track_resources(self, monkeypatch):
+ myflag = [False]
+ def pypy_set_track_resources(flag):
+ myflag[0] = flag
+ monkeypatch.setattr(sys, 'pypy_set_track_resources', pypy_set_track_resources, raising=False)
+ self.check(['-X', 'track-resources'], {}, sys_argv=[''], run_stdin=True)
+ assert myflag[0] == True
class TestInteraction:
"""
@@ -1074,4 +1081,3 @@
# assert it did not crash
finally:
sys.path[:] = old_sys_path
-
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -427,3 +427,28 @@
space.finish()
# assert that we reach this point without getting interrupted
# by the OperationError(NameError)
+
+ def test_format_traceback(self):
+ from pypy.tool.pytest.objspace import maketestobjspace
+ from pypy.interpreter.gateway import interp2app
+ #
+ def format_traceback(space):
+ return space.format_traceback()
+ #
+ space = maketestobjspace()
+ w_format_traceback = space.wrap(interp2app(format_traceback))
+ w_tb = space.appexec([w_format_traceback], """(format_traceback):
+ def foo():
+ return bar()
+ def bar():
+ return format_traceback()
+ return foo()
+ """)
+ tb = space.str_w(w_tb)
+ expected = '\n'.join([
+ ' File "?", line 6, in anonymous', # this is the appexec code object
+ ' File "?", line 3, in foo',
+ ' File "?", line 5, in bar',
+ ''
+ ])
+ assert tb == expected
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
from rpython.rlib import rdynload, clibffi, entrypoint
from rpython.rtyper.lltypesystem import rffi
-VERSION = "1.7.0"
+VERSION = "1.8.0"
FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
try:
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -157,11 +157,13 @@
mustfree_max_plus_1 = 0
buffer = lltype.malloc(rffi.CCHARP.TO, size, flavor='raw')
try:
+ keepalives = [None] * len(args_w) # None or strings
for i in range(len(args_w)):
data = rffi.ptradd(buffer, cif_descr.exchange_args[i])
w_obj = args_w[i]
argtype = self.fargs[i]
- if argtype.convert_argument_from_object(data, w_obj):
+ if argtype.convert_argument_from_object(data, w_obj,
+ keepalives, i):
# argtype is a pointer type, and w_obj a list/tuple/str
mustfree_max_plus_1 = i + 1
@@ -177,9 +179,13 @@
if isinstance(argtype, W_CTypePointer):
data = rffi.ptradd(buffer, cif_descr.exchange_args[i])
flag = get_mustfree_flag(data)
+ raw_cdata = rffi.cast(rffi.CCHARPP, data)[0]
if flag == 1:
- raw_cdata = rffi.cast(rffi.CCHARPP, data)[0]
lltype.free(raw_cdata, flavor='raw')
+ elif flag >= 4:
+ value = keepalives[i]
+ assert value is not None
+ rffi.free_nonmovingbuffer(value, raw_cdata, chr(flag))
lltype.free(buffer, flavor='raw')
keepalive_until_here(args_w)
return w_res
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -83,7 +83,7 @@
raise oefmt(space.w_TypeError, "cannot initialize cdata '%s'",
self.name)
- def convert_argument_from_object(self, cdata, w_ob):
+ def convert_argument_from_object(self, cdata, w_ob, keepalives, i):
self.convert_from_object(cdata, w_ob)
return False
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -14,8 +14,8 @@
class W_CTypePtrOrArray(W_CType):
- _attrs_ = ['ctitem', 'can_cast_anything', 'length']
- _immutable_fields_ = ['ctitem', 'can_cast_anything', 'length']
+ _attrs_ = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
+ _immutable_fields_ = ['ctitem', 'can_cast_anything', 'accept_str', 'length']
length = -1
def __init__(self, space, size, extra, extra_position, ctitem,
@@ -28,6 +28,9 @@
# - for functions, it is the return type
self.ctitem = ctitem
self.can_cast_anything = could_cast_anything and ctitem.cast_anything
+ self.accept_str = (self.can_cast_anything or
+ (ctitem.is_primitive_integer and
+ ctitem.size == rffi.sizeof(lltype.Char)))
def is_unichar_ptr_or_array(self):
return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar)
@@ -70,9 +73,7 @@
pass
else:
self._convert_array_from_listview(cdata, space.listview(w_ob))
- elif (self.can_cast_anything or
- (self.ctitem.is_primitive_integer and
- self.ctitem.size == rffi.sizeof(lltype.Char))):
+ elif self.accept_str:
if not space.isinstance_w(w_ob, space.w_str):
raise self._convert_error("str or list or tuple", w_ob)
s = space.str_w(w_ob)
@@ -260,8 +261,16 @@
else:
return lltype.nullptr(rffi.CCHARP.TO)
- def _prepare_pointer_call_argument(self, w_init, cdata):
+ def _prepare_pointer_call_argument(self, w_init, cdata, keepalives, i):
space = self.space
+ if self.accept_str and space.isinstance_w(w_init, space.w_str):
+ # special case to optimize strings passed to a "char *" argument
+ value = w_init.str_w(space)
+ keepalives[i] = value
+ buf, buf_flag = rffi.get_nonmovingbuffer_final_null(value)
+ rffi.cast(rffi.CCHARPP, cdata)[0] = buf
+ return ord(buf_flag) # 4, 5 or 6
+ #
if (space.isinstance_w(w_init, space.w_list) or
space.isinstance_w(w_init, space.w_tuple)):
length = space.int_w(space.len(w_init))
@@ -297,10 +306,11 @@
rffi.cast(rffi.CCHARPP, cdata)[0] = result
return 1
- def convert_argument_from_object(self, cdata, w_ob):
+ def convert_argument_from_object(self, cdata, w_ob, keepalives, i):
from pypy.module._cffi_backend.ctypefunc import set_mustfree_flag
result = (not isinstance(w_ob, cdataobj.W_CData) and
- self._prepare_pointer_call_argument(w_ob, cdata))
+ self._prepare_pointer_call_argument(w_ob, cdata,
+ keepalives, i))
if result == 0:
self.convert_from_object(cdata, w_ob)
set_mustfree_flag(cdata, result)
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -353,7 +353,7 @@
'array.array' or numpy arrays."""
#
w_ctchara = newtype._new_chara_type(self.space)
- return func.from_buffer(self.space, w_ctchara, w_python_buffer)
+ return func._from_buffer(self.space, w_ctchara, w_python_buffer)
@unwrap_spec(w_arg=W_CData)
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -1,7 +1,8 @@
from rpython.rtyper.annlowlevel import llstr
from rpython.rtyper.lltypesystem import lltype, rffi
from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
-from rpython.rlib.objectmodel import keepalive_until_here
+from rpython.rlib.objectmodel import keepalive_until_here, we_are_translated
+from rpython.rlib import jit
from pypy.interpreter.error import OperationError, oefmt
from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
@@ -132,17 +133,66 @@
raise oefmt(space.w_TypeError,
"needs 'char[]', got '%s'", w_ctype.name)
#
+ return _from_buffer(space, w_ctype, w_x)
+
+def _from_buffer(space, w_ctype, w_x):
buf = _fetch_as_read_buffer(space, w_x)
- try:
- _cdata = buf.get_raw_address()
- except ValueError:
- raise oefmt(space.w_TypeError,
- "from_buffer() got a '%T' object, which supports the "
- "buffer interface but cannot be rendered as a plain "
- "raw address on PyPy", w_x)
+ if space.isinstance_w(w_x, space.w_str):
+ _cdata = get_raw_address_of_string(space, w_x)
+ else:
+ try:
+ _cdata = buf.get_raw_address()
+ except ValueError:
+ raise oefmt(space.w_TypeError,
+ "from_buffer() got a '%T' object, which supports the "
+ "buffer interface but cannot be rendered as a plain "
+ "raw address on PyPy", w_x)
#
return cdataobj.W_CDataFromBuffer(space, _cdata, w_ctype, buf, w_x)
+# ____________________________________________________________
+
+class RawBytes(object):
+ def __init__(self, string):
+ self.ptr = rffi.str2charp(string, track_allocation=False)
+ def __del__(self):
+ rffi.free_charp(self.ptr, track_allocation=False)
+
+class RawBytesCache(object):
+ def __init__(self, space):
+ from pypy.interpreter.baseobjspace import W_Root
+ from rpython.rlib import rweakref
+ self.wdict = rweakref.RWeakKeyDictionary(W_Root, RawBytes)
+
+ at jit.dont_look_inside
+def get_raw_address_of_string(space, w_x):
+ """Special case for ffi.from_buffer(string). Returns a 'char *' that
+ is valid as long as the string object is alive. Two calls to
+ ffi.from_buffer(same_string) are guaranteed to return the same pointer.
+ """
+ from rpython.rtyper.annlowlevel import llstr
+ from rpython.rtyper.lltypesystem.rstr import STR
+ from rpython.rtyper.lltypesystem import llmemory
+ from rpython.rlib import rgc
+
+ cache = space.fromcache(RawBytesCache)
+ rawbytes = cache.wdict.get(w_x)
+ if rawbytes is None:
+ data = space.str_w(w_x)
+ if we_are_translated() and not rgc.can_move(data):
+ lldata = llstr(data)
+ data_start = (llmemory.cast_ptr_to_adr(lldata) +
+ rffi.offsetof(STR, 'chars') +
+ llmemory.itemoffsetof(STR.chars, 0))
+ data_start = rffi.cast(rffi.CCHARP, data_start)
+ data_start[len(data)] = '\x00' # write the final extra null
+ return data_start
+ rawbytes = RawBytes(data)
+ cache.wdict.set(w_x, rawbytes)
+ return rawbytes.ptr
+
+# ____________________________________________________________
+
def unsafe_escaping_ptr_for_ptr_or_array(w_cdata):
if not w_cdata.ctype.is_nonfunc_pointer_or_array:
diff --git a/pypy/module/_cffi_backend/parse_c_type.py b/pypy/module/_cffi_backend/parse_c_type.py
--- a/pypy/module/_cffi_backend/parse_c_type.py
+++ b/pypy/module/_cffi_backend/parse_c_type.py
@@ -97,11 +97,8 @@
[rffi.INT], rffi.CCHARP)
def parse_c_type(info, input):
- p_input = rffi.str2charp(input)
- try:
+ with rffi.scoped_view_charp(input) as p_input:
res = ll_parse_c_type(info, p_input)
- finally:
- rffi.free_charp(p_input)
return rffi.cast(lltype.Signed, res)
NULL_CTX = lltype.nullptr(PCTX.TO)
@@ -130,15 +127,13 @@
return rffi.getintfield(src_ctx, 'c_num_types')
def search_in_globals(ctx, name):
- c_name = rffi.str2charp(name)
- result = ll_search_in_globals(ctx, c_name,
- rffi.cast(rffi.SIZE_T, len(name)))
- rffi.free_charp(c_name)
+ with rffi.scoped_view_charp(name) as c_name:
+ result = ll_search_in_globals(ctx, c_name,
+ rffi.cast(rffi.SIZE_T, len(name)))
return rffi.cast(lltype.Signed, result)
def search_in_struct_unions(ctx, name):
- c_name = rffi.str2charp(name)
- result = ll_search_in_struct_unions(ctx, c_name,
- rffi.cast(rffi.SIZE_T, len(name)))
- rffi.free_charp(c_name)
+ with rffi.scoped_view_charp(name) as c_name:
+ result = ll_search_in_struct_unions(ctx, c_name,
+ rffi.cast(rffi.SIZE_T, len(name)))
return rffi.cast(lltype.Signed, result)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
# ____________________________________________________________
import sys
-assert __version__ == "1.7.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.0", ("This test_c.py file is for testing a version"
" of cffi that differs from the one that we"
" get from 'import _cffi_backend'")
if sys.version_info < (3,):
@@ -3330,13 +3330,18 @@
BChar = new_primitive_type("char")
BCharP = new_pointer_type(BChar)
BCharA = new_array_type(BCharP, None)
- py.test.raises(TypeError, from_buffer, BCharA, b"foo")
+ p1 = from_buffer(BCharA, b"foo")
+ assert p1 == from_buffer(BCharA, b"foo")
+ import gc; gc.collect()
+ assert p1 == from_buffer(BCharA, b"foo")
py.test.raises(TypeError, from_buffer, BCharA, u+"foo")
try:
from __builtin__ import buffer
except ImportError:
pass
else:
+ # from_buffer(buffer(b"foo")) does not work, because it's not
+ # implemented on pypy; only from_buffer(b"foo") works.
py.test.raises(TypeError, from_buffer, BCharA, buffer(b"foo"))
py.test.raises(TypeError, from_buffer, BCharA, buffer(u+"foo"))
try:
diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -38,23 +38,33 @@
errors = None
fd = -1
cffi_fileobj = None # pypy/module/_cffi_backend
+ w_tb = None # String representation of the traceback at creation time
newlines = 0 # Updated when the stream is closed
def __init__(self, space):
self.space = space
self.register_finalizer(space)
+ if self.space.sys.track_resources:
+ self.w_tb = self.space.format_traceback()
def _finalize_(self):
# assume that the file and stream objects are only visible in the
# thread that runs _finalize_, so no race condition should be
# possible and no locking is done here.
- if self.stream is not None:
- try:
- self.direct_close()
- except StreamErrors as e:
- operr = wrap_streamerror(self.space, e, self.w_name)
- raise operr
+ if self.stream is None:
+ return
+ if self.space.sys.track_resources:
+ w_repr = self.space.repr(self)
+ str_repr = self.space.str_w(w_repr)
+ w_msg = self.space.wrap("WARNING: unclosed file: " + str_repr)
+ self.space.resource_warning(w_msg, self.w_tb)
+ #
+ try:
+ self.direct_close()
+ except StreamErrors as e:
+ operr = wrap_streamerror(self.space, e, self.w_name)
+ raise operr
def fdopenstream(self, stream, fd, mode, w_name=None):
self.fd = fd
diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -1,5 +1,6 @@
from __future__ import with_statement
-import py, os, errno
+import pytest, os, errno
+from pypy.interpreter.gateway import interp2app, unwrap_spec
def getfile(space):
return space.appexec([], """():
@@ -10,13 +11,24 @@
return file
""")
+# the following function is used e.g. in test_resource_warning
+ at unwrap_spec(regex=str, s=str)
+def regex_search(space, regex, s):
+ import re
+ import textwrap
+ regex = textwrap.dedent(regex).strip()
+ m = re.search(regex, s)
+ m = bool(m)
+ return space.wrap(m)
+
class AppTestFile(object):
spaceconfig = dict(usemodules=("_file",))
def setup_class(cls):
cls.w_temppath = cls.space.wrap(
- str(py.test.ensuretemp("fileimpl").join("foo.txt")))
+ str(pytest.ensuretemp("fileimpl").join("foo.txt")))
cls.w_file = getfile(cls.space)
+ cls.w_regex_search = cls.space.wrap(interp2app(regex_search))
def test_simple(self):
f = self.file(self.temppath, "w")
@@ -206,6 +218,9 @@
assert exc.value.filename == os.curdir
def test_encoding_errors(self):
+ import sys
+ if '__pypy__' not in sys.builtin_module_names:
+ pytest.skip("pypy only test")
import _file
with self.file(self.temppath, "w") as f:
@@ -254,6 +269,71 @@
if '__pypy__' in sys.builtin_module_names:
assert repr(self.temppath) in g.getvalue()
+ @pytest.mark.skipif("config.option.runappdirect")
+ def test_track_resources(self):
+ import os, gc, sys, cStringIO
+ if '__pypy__' not in sys.builtin_module_names:
+ skip("pypy specific test")
+ def fn(flag1, flag2, do_close=False):
+ sys.pypy_set_track_resources(flag1)
+ f = self.file(self.temppath, 'w')
+ sys.pypy_set_track_resources(flag2)
+ buf = cStringIO.StringIO()
+ preverr = sys.stderr
+ try:
+ sys.stderr = buf
+ if do_close:
+ f.close()
+ del f
+ gc.collect() # force __del__ to be called
+ finally:
+ sys.stderr = preverr
+ sys.pypy_set_track_resources(False)
+ return buf.getvalue()
+
+ # check with track_resources disabled
+ assert fn(False, False) == ""
+ #
+ # check that we don't get the warning if we actually close the file
+ assert fn(False, False, do_close=True) == ""
+ #
+ # check with track_resources enabled
+ msg = fn(True, True)
+ assert self.regex_search(r"""
+ WARNING: unclosed file: <open file .*>
+ Created at \(most recent call last\):
+ File ".*", line .*, in test_track_resources
+ File ".*", line .*, in fn
+ """, msg)
+ #
+ # check with track_resources enabled in the destructor BUT with a
+ # file which was created when track_resources was disabled
+ msg = fn(False, True)
+ assert self.regex_search("WARNING: unclosed file: <open file .*>", msg)
+ assert "Created at" not in msg
+
+ @pytest.mark.skipif("config.option.runappdirect")
+ def test_track_resources_dont_crash(self):
+ import os, gc, sys, cStringIO
+ if '__pypy__' not in sys.builtin_module_names:
+ skip("pypy specific test")
+ #
+ # try hard to create a code object whose co_filename points to an
+ # EXISTING file, so that traceback.py tries to open it when formatting
+ # the stacktrace
+ f = open(self.temppath, 'w')
+ f.close()
+ co = compile('open("%s")' % self.temppath, self.temppath, 'exec')
+ sys.pypy_set_track_resources(True)
+ try:
+ # this exec used to fail, because space.format_traceback tried to
+ # recurively open a file, causing an infinite recursion. For the
+ # purpose of this test, it is enough that it actually finishes
+ # without errors
+ exec co
+ finally:
+ sys.pypy_set_track_resources(False)
+
def test_truncate(self):
f = self.file(self.temppath, "w")
f.write("foo")
@@ -313,7 +393,7 @@
cls.old_read = os.read
if cls.runappdirect:
- py.test.skip("works with internals of _file impl on py.py")
+ pytest.skip("works with internals of _file impl on py.py")
def read(fd, n=None):
if fd != 424242:
return cls.old_read(fd, n)
@@ -352,9 +432,9 @@
def setup_class(cls):
if not cls.runappdirect:
- py.test.skip("likely to deadlock when interpreted by py.py")
+ pytest.skip("likely to deadlock when interpreted by py.py")
cls.w_temppath = cls.space.wrap(
- str(py.test.ensuretemp("fileimpl").join("concurrency.txt")))
+ str(pytest.ensuretemp("fileimpl").join("concurrency.txt")))
cls.w_file = getfile(cls.space)
def test_concurrent_writes(self):
@@ -465,7 +545,7 @@
def setup_class(cls):
cls.w_temppath = cls.space.wrap(
- str(py.test.ensuretemp("fileimpl").join("foo.txt")))
+ str(pytest.ensuretemp("fileimpl").join("foo.txt")))
cls.w_file = getfile(cls.space)
def test___enter__(self):
diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ b/pypy/module/_multiprocessing/interp_connection.py
@@ -401,21 +401,20 @@
_WriteFile, ERROR_NO_SYSTEM_RESOURCES)
from rpython.rlib import rwin32
- charp = rffi.str2charp(buf)
- written_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
- flavor='raw')
- try:
- result = _WriteFile(
- self.handle, rffi.ptradd(charp, offset),
- size, written_ptr, rffi.NULL)
+ with rffi.scoped_view_charp(buf) as charp:
+ written_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
+ flavor='raw')
+ try:
+ result = _WriteFile(
+ self.handle, rffi.ptradd(charp, offset),
+ size, written_ptr, rffi.NULL)
- if (result == 0 and
- rwin32.GetLastError_saved() == ERROR_NO_SYSTEM_RESOURCES):
- raise oefmt(space.w_ValueError,
- "Cannot send %d bytes over connection", size)
- finally:
- rffi.free_charp(charp)
- lltype.free(written_ptr, flavor='raw')
+ if (result == 0 and
+ rwin32.GetLastError_saved() == ERROR_NO_SYSTEM_RESOURCES):
+ raise oefmt(space.w_ValueError,
+ "Cannot send %d bytes over connection", size)
+ finally:
+ lltype.free(written_ptr, flavor='raw')
def do_recv_string(self, space, buflength, maxlength):
from pypy.module._multiprocessing.interp_win32 import (
diff --git a/pypy/module/_rawffi/alt/interp_funcptr.py b/pypy/module/_rawffi/alt/interp_funcptr.py
--- a/pypy/module/_rawffi/alt/interp_funcptr.py
+++ b/pypy/module/_rawffi/alt/interp_funcptr.py
@@ -20,7 +20,8 @@
def _getfunc(space, CDLL, w_name, w_argtypes, w_restype):
argtypes_w, argtypes, w_restype, restype = unpack_argtypes(
space, w_argtypes, w_restype)
- if space.isinstance_w(w_name, space.w_str):
+ if (space.isinstance_w(w_name, space.w_str) or
+ space.isinstance_w(w_name, space.w_unicode)):
name = space.str_w(w_name)
try:
func = CDLL.cdll.getpointer(name, argtypes, restype,
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -151,9 +151,23 @@
class W_Socket(W_Root):
+ w_tb = None # String representation of the traceback at creation time
+
def __init__(self, space, sock):
+ self.space = space
self.sock = sock
register_socket(space, sock)
+ if self.space.sys.track_resources:
+ self.w_tb = self.space.format_traceback()
+ self.register_finalizer(space)
+
+ def _finalize_(self):
+ is_open = self.sock.fd >= 0
+ if is_open and self.space.sys.track_resources:
+ w_repr = self.space.repr(self)
+ str_repr = self.space.str_w(w_repr)
+ w_msg = self.space.wrap("WARNING: unclosed " + str_repr)
+ self.space.resource_warning(w_msg, self.w_tb)
def get_type_w(self, space):
return space.wrap(self.sock.type)
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -1,6 +1,8 @@
import sys, os
-import py
+import pytest
from pypy.tool.pytest.objspace import gettestobjspace
+from pypy.interpreter.gateway import interp2app
+from pypy.module._file.test.test_file import regex_search
from rpython.tool.udir import udir
from rpython.rlib import rsocket
from rpython.rtyper.lltypesystem import lltype, rffi
@@ -12,8 +14,6 @@
mod.w_socket = space.appexec([], "(): import _socket as m; return m")
mod.path = udir.join('fd')
mod.path.write('fo')
- mod.raises = py.test.raises # make raises available from app-level tests
- mod.skip = py.test.skip
def test_gethostname():
host = space.appexec([w_socket], "(_socket): return _socket.gethostname()")
@@ -41,7 +41,7 @@
for host in ["localhost", "127.0.0.1", "::1"]:
if host == "::1" and not ipv6:
from pypy.interpreter.error import OperationError
- with py.test.raises(OperationError):
+ with pytest.raises(OperationError):
space.appexec([w_socket, space.wrap(host)],
"(_socket, host): return _socket.gethostbyaddr(host)")
continue
@@ -57,14 +57,14 @@
assert space.unwrap(port) == 25
# 1 arg version
if sys.version_info < (2, 4):
- py.test.skip("getservbyname second argument is not optional before python 2.4")
+ pytest.skip("getservbyname second argument is not optional before python 2.4")
port = space.appexec([w_socket, space.wrap(name)],
"(_socket, name): return _socket.getservbyname(name)")
assert space.unwrap(port) == 25
def test_getservbyport():
if sys.version_info < (2, 4):
- py.test.skip("getservbyport does not exist before python 2.4")
+ pytest.skip("getservbyport does not exist before python 2.4")
port = 25
# 2 args version
name = space.appexec([w_socket, space.wrap(port)],
@@ -97,7 +97,7 @@
def test_fromfd():
# XXX review
if not hasattr(socket, 'fromfd'):
- py.test.skip("No socket.fromfd on this platform")
+ pytest.skip("No socket.fromfd on this platform")
orig_fd = path.open()
fd = space.appexec([w_socket, space.wrap(orig_fd.fileno()),
space.wrap(socket.AF_INET), space.wrap(socket.SOCK_STREAM),
@@ -157,7 +157,7 @@
def test_pton_ntop_ipv4():
if not hasattr(socket, 'inet_pton'):
- py.test.skip('No socket.inet_pton on this platform')
+ pytest.skip('No socket.inet_pton on this platform')
tests = [
("123.45.67.89", "\x7b\x2d\x43\x59"),
("0.0.0.0", "\x00" * 4),
@@ -173,9 +173,9 @@
def test_ntop_ipv6():
if not hasattr(socket, 'inet_pton'):
- py.test.skip('No socket.inet_pton on this platform')
+ pytest.skip('No socket.inet_pton on this platform')
if not socket.has_ipv6:
- py.test.skip("No IPv6 on this platform")
+ pytest.skip("No IPv6 on this platform")
tests = [
("\x00" * 16, "::"),
("\x01" * 16, ":".join(["101"] * 8)),
@@ -194,9 +194,9 @@
def test_pton_ipv6():
if not hasattr(socket, 'inet_pton'):
- py.test.skip('No socket.inet_pton on this platform')
+ pytest.skip('No socket.inet_pton on this platform')
if not socket.has_ipv6:
- py.test.skip("No IPv6 on this platform")
+ pytest.skip("No IPv6 on this platform")
tests = [
("\x00" * 16, "::"),
("\x01" * 16, ":".join(["101"] * 8)),
@@ -215,7 +215,7 @@
assert space.unwrap(w_packed) == packed
def test_has_ipv6():
- py.test.skip("has_ipv6 is always True on PyPy for now")
+ pytest.skip("has_ipv6 is always True on PyPy for now")
res = space.appexec([w_socket], "(_socket): return _socket.has_ipv6")
assert space.unwrap(res) == socket.has_ipv6
@@ -229,7 +229,7 @@
w_l = space.appexec([w_socket, space.wrap(host), space.wrap(port)],
"(_socket, host, port): return _socket.getaddrinfo(host, long(port))")
assert space.unwrap(w_l) == info
- py.test.skip("Unicode conversion is too slow")
+ pytest.skip("Unicode conversion is too slow")
w_l = space.appexec([w_socket, space.wrap(unicode(host)), space.wrap(port)],
"(_socket, host, port): return _socket.getaddrinfo(host, port)")
assert space.unwrap(w_l) == info
@@ -250,7 +250,7 @@
def test_addr_raw_packet():
from pypy.module._socket.interp_socket import addr_as_object
if not hasattr(rsocket._c, 'sockaddr_ll'):
- py.test.skip("posix specific test")
+ pytest.skip("posix specific test")
# HACK: To get the correct interface number of lo, which in most cases is 1,
# but can be anything (i.e. 39), we need to call the libc function
# if_nametoindex to get the correct index
@@ -314,6 +314,7 @@
def setup_class(cls):
cls.space = space
cls.w_udir = space.wrap(str(udir))
+ cls.w_regex_search = space.wrap(interp2app(regex_search))
def teardown_class(cls):
if not cls.runappdirect:
@@ -402,6 +403,64 @@
if os.name != 'nt':
raises(OSError, os.close, fileno)
+ def test_socket_track_resources(self):
+ import _socket, os, gc, sys, cStringIO
+ s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0)
+ fileno = s.fileno()
+ assert s.fileno() >= 0
+ s.close()
+ assert s.fileno() < 0
+ s.close()
+ if os.name != 'nt':
+ raises(OSError, os.close, fileno)
+
+ @pytest.mark.skipif("config.option.runappdirect")
+ def test_track_resources(self):
+ import os, gc, sys, cStringIO
+ import _socket
+ if '__pypy__' not in sys.builtin_module_names:
+ skip("pypy specific test")
+ #
+ def fn(flag1, flag2, do_close=False):
+ sys.pypy_set_track_resources(flag1)
+ mysock = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0)
+ sys.pypy_set_track_resources(flag2)
+ buf = cStringIO.StringIO()
+ preverr = sys.stderr
+ try:
+ sys.stderr = buf
+ if do_close:
+ mysock.close()
+ del mysock
+ gc.collect() # force __del__ to be called
+ finally:
+ sys.stderr = preverr
+ sys.pypy_set_track_resources(False)
+ return buf.getvalue()
+
+ # check with track_resources disabled
+ assert fn(False, False) == ""
+ #
+ # check that we don't get the warning if we actually closed the socket
+ msg = fn(True, True, do_close=True)
+ assert msg == ''
+ #
+ # check with track_resources enabled
+ msg = fn(True, True)
+ assert self.regex_search(r"""
+ WARNING: unclosed <socket object, .*>
+ Created at \(most recent call last\):
+ File ".*", line .*, in test_track_resources
+ File ".*", line .*, in fn
+ """, msg)
+ #
+ # track_resources is enabled after the construction of the socket. in
+ # this case, the socket is not registered for finalization at all, so
+ # we don't see a message
+ msg = fn(False, True)
+ assert msg == ''
+
+
def test_socket_close_error(self):
import _socket, os
if os.name == 'nt':
@@ -630,11 +689,11 @@
class AppTestNetlink:
def setup_class(cls):
if not hasattr(os, 'getpid'):
- py.test.skip("AF_NETLINK needs os.getpid()")
+ pytest.skip("AF_NETLINK needs os.getpid()")
w_ok = space.appexec([], "(): import _socket; " +
"return hasattr(_socket, 'AF_NETLINK')")
if not space.is_true(w_ok):
- py.test.skip("no AF_NETLINK on this platform")
+ pytest.skip("no AF_NETLINK on this platform")
cls.space = space
def test_connect_to_kernel_netlink_routing_socket(self):
@@ -650,11 +709,11 @@
class AppTestPacket:
def setup_class(cls):
if not hasattr(os, 'getuid') or os.getuid() != 0:
- py.test.skip("AF_PACKET needs to be root for testing")
+ pytest.skip("AF_PACKET needs to be root for testing")
w_ok = space.appexec([], "(): import _socket; " +
"return hasattr(_socket, 'AF_PACKET')")
if not space.is_true(w_ok):
- py.test.skip("no AF_PACKET on this platform")
+ pytest.skip("no AF_PACKET on this platform")
cls.space = space
def test_convert_between_tuple_and_sockaddr_ll(self):
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -135,7 +135,7 @@
def __init__(self, ctx, protos):
self.protos = protos
- self.buf, self.pinned, self.is_raw = rffi.get_nonmovingbuffer(protos)
+ self.buf, self.bufflag = rffi.get_nonmovingbuffer(protos)
NPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
# set both server and client callbacks, because the context
@@ -147,7 +147,7 @@
def __del__(self):
rffi.free_nonmovingbuffer(
- self.protos, self.buf, self.pinned, self.is_raw)
+ self.protos, self.buf, self.bufflag)
@staticmethod
def advertiseNPN_cb(s, data_ptr, len_ptr, args):
@@ -181,7 +181,7 @@
def __init__(self, ctx, protos):
self.protos = protos
- self.buf, self.pinned, self.is_raw = rffi.get_nonmovingbuffer(protos)
+ self.buf, self.bufflag = rffi.get_nonmovingbuffer(protos)
ALPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
with rffi.scoped_str2charp(protos) as protos_buf:
@@ -193,7 +193,7 @@
def __del__(self):
rffi.free_nonmovingbuffer(
- self.protos, self.buf, self.pinned, self.is_raw)
+ self.protos, self.buf, self.bufflag)
@staticmethod
def selectALPN_cb(s, out_ptr, outlen_ptr, client, client_len, args):
@@ -228,7 +228,7 @@
Mix string into the OpenSSL PRNG state. entropy (a float) is a lower
bound on the entropy contained in string."""
- with rffi.scoped_str2charp(string) as buf:
+ with rffi.scoped_nonmovingbuffer(string) as buf:
libssl_RAND_add(buf, len(string), entropy)
def RAND_status(space):
diff --git a/pypy/module/cppyy/capi/builtin_capi.py b/pypy/module/cppyy/capi/builtin_capi.py
--- a/pypy/module/cppyy/capi/builtin_capi.py
+++ b/pypy/module/cppyy/capi/builtin_capi.py
@@ -537,9 +537,8 @@
releasegil=ts_helper,
compilation_info=backend.eci)
def c_charp2stdstring(space, svalue):
- charp = rffi.str2charp(svalue)
- result = _c_charp2stdstring(charp)
- rffi.free_charp(charp)
+ with rffi.scoped_view_charp(svalue) as charp:
+ result = _c_charp2stdstring(charp)
return result
_c_stdstring2stdstring = rffi.llexternal(
"cppyy_stdstring2stdstring",
diff --git a/pypy/module/cppyy/capi/cint_capi.py b/pypy/module/cppyy/capi/cint_capi.py
--- a/pypy/module/cppyy/capi/cint_capi.py
+++ b/pypy/module/cppyy/capi/cint_capi.py
@@ -82,9 +82,8 @@
releasegil=ts_helper,
compilation_info=eci)
def c_charp2TString(space, svalue):
- charp = rffi.str2charp(svalue)
- result = _c_charp2TString(charp)
- rffi.free_charp(charp)
+ with rffi.scoped_view_charp(svalue) as charp:
+ result = _c_charp2TString(charp)
return result
_c_TString2TString = rffi.llexternal(
"cppyy_TString2TString",
diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py
--- a/pypy/module/cppyy/capi/loadable_capi.py
+++ b/pypy/module/cppyy/capi/loadable_capi.py
@@ -65,6 +65,7 @@
else: # only other use is sring
n = len(obj._string)
assert raw_string == rffi.cast(rffi.CCHARP, 0)
+ # XXX could use rffi.get_nonmovingbuffer_final_null()
raw_string = rffi.str2charp(obj._string)
data = rffi.cast(rffi.CCHARPP, data)
data[0] = raw_string
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -11,6 +11,9 @@
from rpython.rtyper.annlowlevel import llhelper
from rpython.rlib.objectmodel import we_are_translated, keepalive_until_here
from rpython.rlib.objectmodel import dont_inline
+from rpython.rlib.rfile import (FILEP, c_fread, c_fclose, c_fwrite,
+ c_fdopen, c_fileno,
+ c_fopen)# for tests
from rpython.translator import cdir
from rpython.translator.tool.cbuild import ExternalCompilationInfo
from rpython.translator.gensupp import NameManager
@@ -85,44 +88,32 @@
assert CONST_WSTRING == rffi.CWCHARP
# FILE* interface
-FILEP = rffi.COpaquePtr('FILE')
if sys.platform == 'win32':
dash = '_'
else:
dash = ''
-fileno = rffi.llexternal(dash + 'fileno', [FILEP], rffi.INT)
-fopen = rffi.llexternal('fopen', [CONST_STRING, CONST_STRING], FILEP)
-fdopen = rffi.llexternal(dash + 'fdopen', [rffi.INT, CONST_STRING],
- FILEP, save_err=rffi.RFFI_SAVE_ERRNO)
-_fclose = rffi.llexternal('fclose', [FILEP], rffi.INT)
def fclose(fp):
- if not is_valid_fd(fileno(fp)):
+ if not is_valid_fd(c_fileno(fp)):
return -1
- return _fclose(fp)
+ return c_fclose(fp)
-_fwrite = rffi.llexternal('fwrite',
- [rffi.VOIDP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
- rffi.SIZE_T)
def fwrite(buf, sz, n, fp):
- validate_fd(fileno(fp))
- return _fwrite(buf, sz, n, fp)
+ validate_fd(c_fileno(fp))
+ return c_fwrite(buf, sz, n, fp)
-_fread = rffi.llexternal('fread',
- [rffi.VOIDP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
- rffi.SIZE_T)
def fread(buf, sz, n, fp):
- validate_fd(fileno(fp))
- return _fread(buf, sz, n, fp)
+ validate_fd(c_fileno(fp))
+ return c_fread(buf, sz, n, fp)
_feof = rffi.llexternal('feof', [FILEP], rffi.INT)
def feof(fp):
- validate_fd(fileno(fp))
+ validate_fd(c_fileno(fp))
return _feof(fp)
def is_valid_fp(fp):
- return is_valid_fd(fileno(fp))
+ return is_valid_fd(c_fileno(fp))
pypy_decl = 'pypy_decl.h'
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -96,7 +96,8 @@
raise oefmt(space.w_ValueError,
"bytes_attach called on object with ob_size %d but trying to store %d",
py_str.c_ob_size, len(s))
- rffi.c_memcpy(py_str.c_ob_sval, rffi.str2charp(s), len(s))
+ with rffi.scoped_nonmovingbuffer(s) as s_ptr:
+ rffi.c_memcpy(py_str.c_ob_sval, s_ptr, len(s))
py_str.c_ob_sval[len(s)] = '\0'
py_str.c_ob_shash = space.hash_w(w_obj)
py_str.c_ob_sstate = rffi.cast(rffi.INT, 1) # SSTATE_INTERNED_MORTAL
diff --git a/pypy/module/cpyext/pyfile.py b/pypy/module/cpyext/pyfile.py
--- a/pypy/module/cpyext/pyfile.py
+++ b/pypy/module/cpyext/pyfile.py
@@ -1,6 +1,7 @@
from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.rfile import c_setvbuf, _IONBF
from pypy.module.cpyext.api import (
- cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, build_type_checkers, fdopen)
+ cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, build_type_checkers, c_fdopen)
from pypy.module.cpyext.pyobject import PyObject
from pypy.module.cpyext.object import Py_PRINT_RAW
from pypy.interpreter.error import (OperationError, oefmt,
@@ -64,11 +65,12 @@
if (fd < 0 or not mode or mode[0] not in ['r', 'w', 'a', 'U'] or
('U' in mode and ('w' in mode or 'a' in mode))):
raise oefmt(space.w_IOError, 'invalid fileno or mode')
- ret = fdopen(fd, mode)
+ ret = c_fdopen(fd, mode)
if not ret:
raise exception_from_saved_errno(space, space.w_IOError)
+ # XXX fix this once use-file-star-for-file lands
+ c_setvbuf(ret, lltype.nullptr(rffi.CCHARP.TO), _IONBF, 0)
return ret
-
@cpython_api([FILEP, CONST_STRING, CONST_STRING, rffi.VOIDP], PyObject)
def PyFile_FromFile(space, fp, name, mode, close):
diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -3,7 +3,7 @@
from pypy.module.cpyext.test.test_api import BaseApiTest
from pypy.module.cpyext.eval import (
Py_single_input, Py_file_input, Py_eval_input, PyCompilerFlags)
-from pypy.module.cpyext.api import fopen, fclose, fileno, Py_ssize_tP
+from pypy.module.cpyext.api import c_fopen, c_fclose, c_fileno, Py_ssize_tP
from pypy.interpreter.gateway import interp2app
from pypy.interpreter.astcompiler import consts
from rpython.tool.udir import udir
@@ -130,19 +130,19 @@
def test_run_file(self, space, api):
filepath = udir / "cpyext_test_runfile.py"
filepath.write("raise ZeroDivisionError")
- fp = fopen(str(filepath), "rb")
+ fp = c_fopen(str(filepath), "rb")
filename = rffi.str2charp(str(filepath))
w_globals = w_locals = space.newdict()
api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals)
- fclose(fp)
+ c_fclose(fp)
assert api.PyErr_Occurred() is space.w_ZeroDivisionError
api.PyErr_Clear()
# try again, but with a closed file
- fp = fopen(str(filepath), "rb")
- os.close(fileno(fp))
+ fp = c_fopen(str(filepath), "rb")
+ os.close(c_fileno(fp))
api.PyRun_File(fp, filename, Py_file_input, w_globals, w_locals)
- fclose(fp)
+ c_fclose(fp)
assert api.PyErr_Occurred() is space.w_IOError
api.PyErr_Clear()
diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py
--- a/pypy/module/cpyext/test/test_pyfile.py
+++ b/pypy/module/cpyext/test/test_pyfile.py
@@ -1,5 +1,4 @@
from pypy.conftest import option
-from pypy.module.cpyext.api import fopen, fclose, fwrite
from pypy.module.cpyext.test.test_api import BaseApiTest
from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
from pypy.module.cpyext.object import Py_PRINT_RAW
@@ -133,6 +132,15 @@
return PyLong_FromLong(0);
return PyLong_FromLong(ftell(fp));
"""),
+ ("read_10", "METH_O",
+ """
+ char s[10];
+ FILE * fp = PyFile_AsFile(args);
+ if (fp == NULL)
+ return PyLong_FromLong(0);
+ fread(s, 1, 10, fp);
+ return PyLong_FromLong(ftell(fp));
+ """),
])
filename = self.udir + "/_test_file"
with open(filename, 'w') as fid:
@@ -142,5 +150,12 @@
t_py = fid.tell()
assert t_py == 80
t_c = module.get_c_tell(fid)
- assert t_c == t_py
+ assert t_c == t_py
+ print '-------- tell ',t_c
+ t_c = module.read_10(fid)
+ assert t_c == t_py + 10
+ print '-------- tell ',t_c
+ t_py = fid.tell()
+ assert t_c == t_py, 'after a fread, c level ftell(fp) %d but PyFile.tell() %d' % (t_c, t_py)
+
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -20,6 +20,7 @@
self.defaultencoding = "ascii"
self.filesystemencoding = None
self.debug = True
+ self.track_resources = False
self.dlopenflags = rdynload._dlopen_default_mode()
interpleveldefs = {
@@ -55,6 +56,8 @@
'_current_frames' : 'currentframes._current_frames',
'setrecursionlimit' : 'vm.setrecursionlimit',
'getrecursionlimit' : 'vm.getrecursionlimit',
+ 'pypy_set_track_resources' : 'vm.set_track_resources',
+ 'pypy_get_track_resources' : 'vm.get_track_resources',
'setcheckinterval' : 'vm.setcheckinterval',
'getcheckinterval' : 'vm.getcheckinterval',
'exc_info' : 'vm.exc_info',
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -61,6 +61,13 @@
"""
return space.wrap(space.sys.recursionlimit)
+ at unwrap_spec(flag=bool)
+def set_track_resources(space, flag):
+ space.sys.track_resources = flag
+
+def get_track_resources(space):
+ return space.wrap(space.sys.track_resources)
+
@unwrap_spec(interval=int)
def setcheckinterval(space, interval):
"""Tell the Python interpreter to check for asynchronous events every
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
@@ -130,7 +130,7 @@
cls.module = str(udir.join('testownlib.dll'))
else:
subprocess.check_call(
- 'gcc testownlib.c -shared -fPIC -o testownlib.so',
+ 'cc testownlib.c -shared -fPIC -o testownlib.so',
cwd=str(udir), shell=True)
cls.module = str(udir.join('testownlib.so'))
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -852,9 +852,12 @@
assert str(e2.value) == "foo0() takes no arguments (2 given)"
assert str(e3.value) == "foo1() takes exactly one argument (0 given)"
assert str(e4.value) == "foo1() takes exactly one argument (2 given)"
- assert str(e5.value) == "foo2() takes exactly 2 arguments (0 given)"
- assert str(e6.value) == "foo2() takes exactly 2 arguments (1 given)"
- assert str(e7.value) == "foo2() takes exactly 2 arguments (3 given)"
+ assert str(e5.value) in ["foo2 expected 2 arguments, got 0",
+ "foo2() takes exactly 2 arguments (0 given)"]
+ assert str(e6.value) in ["foo2 expected 2 arguments, got 1",
+ "foo2() takes exactly 2 arguments (1 given)"]
+ assert str(e7.value) in ["foo2 expected 2 arguments, got 3",
+ "foo2() takes exactly 2 arguments (3 given)"]
def test_address_of_function():
ffi = FFI()
@@ -1916,3 +1919,47 @@
ffi.cdef("bool f(void);")
lib = verify(ffi, "test_bool_in_cpp", "char f(void) { return 2; }")
assert lib.f() == 1
+
+def test_bool_in_cpp_2():
+ ffi = FFI()
+ ffi.cdef('int add(int a, int b);')
+ lib = verify(ffi, "test_bool_bug_cpp", '''
+ typedef bool _Bool; /* there is a Windows header with this line */
+ int add(int a, int b)
+ {
+ return a + b;
+ }''', source_extension='.cpp')
+ c = lib.add(2, 3)
+ assert c == 5
+
+def test_struct_field_opaque():
+ ffi = FFI()
+ ffi.cdef("struct a { struct b b; };")
+ e = py.test.raises(TypeError, verify,
+ ffi, "test_struct_field_opaque", "?")
+ assert str(e.value) == ("struct a: field 'a.b' is of an opaque"
+ " type (not declared in cdef())")
+ ffi = FFI()
+ ffi.cdef("struct a { struct b b[2]; };")
+ e = py.test.raises(TypeError, verify,
+ ffi, "test_struct_field_opaque", "?")
+ assert str(e.value) == ("struct a: field 'a.b' is of an opaque"
+ " type (not declared in cdef())")
+ ffi = FFI()
+ ffi.cdef("struct a { struct b b[]; };")
+ e = py.test.raises(TypeError, verify,
+ ffi, "test_struct_field_opaque", "?")
+ assert str(e.value) == ("struct a: field 'a.b' is of an opaque"
+ " type (not declared in cdef())")
+
+def test_function_arg_opaque():
+ py.test.skip("can currently declare a function with an opaque struct "
+ "as argument, but AFAICT it's impossible to call it later")
+
+def test_function_returns_opaque():
+ ffi = FFI()
+ ffi.cdef("struct a foo(int);")
+ e = py.test.raises(TypeError, verify,
+ ffi, "test_function_returns_opaque", "?")
+ assert str(e.value) == ("function foo: 'struct a' is used as result type,"
+ " but is opaque")
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
@@ -133,6 +133,12 @@
# You cannot assing character format codes as restype any longer
raises(TypeError, setattr, f, "restype", "i")
+ def test_unicode_function_name(self):
+ f = dll[u'_testfunc_i_bhilfd']
+ f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double]
+ f.restype = c_int
+ result = f(1, 2, 3, 4, 5.0, 6.0)
+ assert result == 21
def test_truncate_python_longs(self):
f = dll._testfunc_i_bhilfd
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -428,4 +428,5 @@
FakeObjSpace.sys.filesystemencoding = 'foobar'
FakeObjSpace.sys.defaultencoding = 'ascii'
FakeObjSpace.sys.dlopenflags = 123
+FakeObjSpace.sys.track_resources = False
FakeObjSpace.builtin = FakeModule()
diff --git a/requirements.txt b/requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,3 @@
-# hypothesis is used for test generation on untranslated jit tests
+# hypothesis is used for test generation on untranslated tests
hypothesis
enum34>=1.1.2
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -883,6 +883,7 @@
ofs_items, itemsize, _ = symbolic.get_array_token(rstr.STR,
self.cpu.translate_support_code)
assert itemsize == 1
+ ofs_items -= 1 # for the extra null character
scale = 0
self._gen_address(resloc, baseloc, ofsloc, scale, ofs_items)
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -280,7 +280,7 @@
concrete_type = '\x00'
def __init__(self, basesize, itemsize, lendescr, flag, is_pure=False, concrete_type='\x00'):
- self.basesize = basesize
+ self.basesize = basesize # this includes +1 for STR
self.itemsize = itemsize
self.lendescr = lendescr # or None, if no length
self.flag = flag
@@ -676,7 +676,7 @@
def unpack_arraydescr(arraydescr):
assert isinstance(arraydescr, ArrayDescr)
- ofs = arraydescr.basesize
+ ofs = arraydescr.basesize # this includes +1 for STR
size = arraydescr.itemsize
sign = arraydescr.is_item_signed()
return size, ofs, sign
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -293,6 +293,7 @@
basesize, itemsize, ofs_length = get_array_token(rstr.STR,
self.cpu.translate_support_code)
assert itemsize == 1
+ basesize -= 1 # for the extra null character
self.emit_gc_load_or_indexed(op, op.getarg(0), op.getarg(1),
itemsize, itemsize, basesize, NOT_SIGNED)
elif opnum == rop.UNICODEGETITEM:
@@ -304,6 +305,7 @@
basesize, itemsize, ofs_length = get_array_token(rstr.STR,
self.cpu.translate_support_code)
assert itemsize == 1
+ basesize -= 1 # for the extra null character
self.emit_gc_store_or_indexed(op, op.getarg(0), op.getarg(1), op.getarg(2),
itemsize, itemsize, basesize)
elif opnum == rop.UNICODESETITEM:
diff --git a/rpython/jit/backend/llsupport/symbolic.py b/rpython/jit/backend/llsupport/symbolic.py
--- a/rpython/jit/backend/llsupport/symbolic.py
+++ b/rpython/jit/backend/llsupport/symbolic.py
@@ -29,7 +29,7 @@
def get_array_token(T, translate_support_code):
# T can be an array or a var-sized structure
if translate_support_code:
- basesize = llmemory.sizeof(T, 0)
+ basesize = llmemory.sizeof(T, 0) # this includes +1 for STR
if isinstance(T, lltype.Struct):
SUBARRAY = getattr(T, T._arrayfld)
itemsize = llmemory.sizeof(SUBARRAY.OF)
@@ -57,6 +57,7 @@
assert carray.length.size == WORD
ofs_length = before_array_part + carray.length.offset
basesize = before_array_part + carray.items.offset
+ basesize += T._hints.get('extra_item_after_alloc', 0) # +1 for STR
carrayitem = ll2ctypes.get_ctypes_type(T.OF)
itemsize = ctypes.sizeof(carrayitem)
return basesize, itemsize, ofs_length
diff --git a/rpython/jit/backend/llsupport/test/test_descr.py b/rpython/jit/backend/llsupport/test/test_descr.py
--- a/rpython/jit/backend/llsupport/test/test_descr.py
+++ b/rpython/jit/backend/llsupport/test/test_descr.py
@@ -435,8 +435,10 @@
def test_bytearray_descr():
c0 = GcCache(False)
descr = get_array_descr(c0, rstr.STR) # for bytearray
+ # note that we get a basesize that has 1 extra byte for the final null char
+ # (only for STR)
assert descr.flag == FLAG_UNSIGNED
- assert descr.basesize == struct.calcsize("PP") # hash, length
+ assert descr.basesize == struct.calcsize("PP") + 1 # hash, length, extra
assert descr.lendescr.offset == struct.calcsize("P") # hash
assert not descr.is_array_of_pointers()
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -647,6 +647,9 @@
""")
def test_rewrite_assembler_newstr_newunicode(self):
+ # note: strdescr.basesize already contains the extra final character,
+ # so that's why newstr(14) is rounded up to 'basesize+15' and not
+ # 'basesize+16'.
self.check_rewrite("""
[i2]
p0 = newstr(14)
@@ -657,12 +660,12 @@
""", """
[i2]
p0 = call_malloc_nursery( \
- %(strdescr.basesize + 16 * strdescr.itemsize + \
+ %(strdescr.basesize + 15 * strdescr.itemsize + \
unicodedescr.basesize + 10 * unicodedescr.itemsize)d)
gc_store(p0, 0, %(strdescr.tid)d, %(tiddescr.field_size)s)
gc_store(p0, %(strlendescr.offset)s, 14, %(strlendescr.field_size)s)
gc_store(p0, 0, 0, %(strhashdescr.field_size)s)
- p1 = nursery_ptr_increment(p0, %(strdescr.basesize + 16 * strdescr.itemsize)d)
+ p1 = nursery_ptr_increment(p0, %(strdescr.basesize + 15 * strdescr.itemsize)d)
gc_store(p1, 0, %(unicodedescr.tid)d, %(tiddescr.field_size)s)
gc_store(p1, %(unicodelendescr.offset)s, 10, %(unicodelendescr.field_size)s)
gc_store(p1, 0, 0, %(unicodehashdescr.field_size)s)
@@ -1240,14 +1243,14 @@
# 'i3 = gc_load_i(p0,i5,%(unicodedescr.itemsize)d)'],
[True, (4,), 'i3 = strgetitem(p0,i1)' '->'
'i3 = gc_load_indexed_i(p0,i1,1,'
- '%(strdescr.basesize)d,1)'],
+ '%(strdescr.basesize-1)d,1)'],
#[False, (4,), 'i3 = strgetitem(p0,i1)' '->'
- # 'i5 = int_add(i1, %(strdescr.basesize)d);'
+ # 'i5 = int_add(i1, %(strdescr.basesize-1)d);'
# 'i3 = gc_load_i(p0,i5,1)'],
## setitem str/unicode
[True, (4,), 'i3 = strsetitem(p0,i1,0)' '->'
'i3 = gc_store_indexed(p0,i1,0,1,'
- '%(strdescr.basesize)d,1)'],
+ '%(strdescr.basesize-1)d,1)'],
[True, (2,4), 'i3 = unicodesetitem(p0,i1,0)' '->'
'i3 = gc_store_indexed(p0,i1,0,'
'%(unicodedescr.itemsize)d,'
diff --git a/rpython/jit/backend/llsupport/test/ztranslation_test.py b/rpython/jit/backend/llsupport/test/ztranslation_test.py
--- a/rpython/jit/backend/llsupport/test/ztranslation_test.py
+++ b/rpython/jit/backend/llsupport/test/ztranslation_test.py
@@ -3,7 +3,7 @@
from rpython.rlib.jit import JitDriver, unroll_parameters, set_param
from rpython.rlib.jit import PARAMETERS, dont_look_inside
from rpython.rlib.jit import promote, _get_virtualizable_token
-from rpython.rlib import jit_hooks, rposix
+from rpython.rlib import jit_hooks, rposix, rgc
from rpython.rlib.objectmodel import keepalive_until_here
from rpython.rlib.rthread import ThreadLocalReference, ThreadLocalField
from rpython.jit.backend.detect_cpu import getcpuclass
@@ -11,7 +11,7 @@
from rpython.jit.codewriter.policy import StopAtXPolicy
from rpython.config.config import ConfigError
from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.lltypesystem import lltype, rffi, rstr
from rpython.rlib.rjitlog import rjitlog as jl
@@ -29,6 +29,7 @@
# - floats neg and abs
# - cast_int_to_float
# - llexternal with macro=True
+ # - extra place for the zero after STR instances
class BasicFrame(object):
_virtualizable_ = ['i']
@@ -56,7 +57,7 @@
return ("/home.py",0,0)
jitdriver = JitDriver(greens = [],
- reds = ['total', 'frame', 'j'],
+ reds = ['total', 'frame', 'prev_s', 'j'],
virtualizables = ['frame'],
get_location = get_location)
def f(i, j):
@@ -68,9 +69,12 @@
total = 0
frame = Frame(i)
j = float(j)
+ prev_s = rstr.mallocstr(16)
while frame.i > 3:
- jitdriver.can_enter_jit(frame=frame, total=total, j=j)
- jitdriver.jit_merge_point(frame=frame, total=total, j=j)
+ jitdriver.can_enter_jit(frame=frame, total=total, j=j,
+ prev_s=prev_s)
+ jitdriver.jit_merge_point(frame=frame, total=total, j=j,
+ prev_s=prev_s)
_get_virtualizable_token(frame)
total += frame.i
if frame.i >= 20:
@@ -82,6 +86,11 @@
k = myabs1(myabs2(j))
if k - abs(j): raise ValueError
if k - abs(-j): raise ValueError
+ s = rstr.mallocstr(16)
+ rgc.ll_write_final_null_char(s)
+ rgc.ll_write_final_null_char(prev_s)
+ if (frame.i & 3) == 0:
+ prev_s = s
return chr(total % 253)
#
class Virt2(object):
diff --git a/rpython/jit/backend/ppc/opassembler.py b/rpython/jit/backend/ppc/opassembler.py
--- a/rpython/jit/backend/ppc/opassembler.py
+++ b/rpython/jit/backend/ppc/opassembler.py
@@ -994,6 +994,7 @@
basesize, itemsize, _ = symbolic.get_array_token(rstr.STR,
self.cpu.translate_support_code)
assert itemsize == 1
+ basesize -= 1 # for the extra null character
scale = 0
self._emit_load_for_copycontent(r.r0, src_ptr_loc, src_ofs_loc, scale)
diff --git a/rpython/jit/backend/test/test_rvmprof.py b/rpython/jit/backend/test/test_rvmprof.py
--- a/rpython/jit/backend/test/test_rvmprof.py
+++ b/rpython/jit/backend/test/test_rvmprof.py
@@ -2,7 +2,8 @@
from rpython.rlib import jit
from rpython.rtyper.annlowlevel import llhelper
from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib.rvmprof import cintf
+from rpython.rlib.rvmprof import cintf, vmprof_execute_code, register_code,\
+ register_code_object_class, _get_vmprof
from rpython.jit.backend.x86.arch import WORD
from rpython.jit.codewriter.policy import JitPolicy
@@ -14,6 +15,7 @@
def helper():
stack = cintf.vmprof_tl_stack.getraw()
+ print stack
if stack:
# not during tracing
visited.append(stack.c_value)
@@ -22,15 +24,34 @@
llfn = llhelper(lltype.Ptr(lltype.FuncType([], lltype.Void)), helper)
- driver = jit.JitDriver(greens=[], reds='auto')
+ driver = jit.JitDriver(greens=['code'], reds='auto')
- def f(n):
+ class CodeObj(object):
+ pass
+
+ def get_code_fn(code, arg):
+ return code
+
+ def get_name(code):
+ return "foo"
+
+ register_code_object_class(CodeObj, get_name)
+
+ @vmprof_execute_code("main", get_code_fn)
+ def f(code, n):
i = 0
while i < n:
- driver.jit_merge_point()
+ driver.jit_merge_point(code=code)
i += 1
llfn()
+ def main(n):
+ cintf.vmprof_tl_stack.setraw(null) # make it empty
+ vmprof = _get_vmprof()
+ code = CodeObj()
+ register_code(code, get_name)
+ return f(code, n)
+
class Hooks(jit.JitHookInterface):
def after_compile(self, debug_info):
self.raw_start = debug_info.asminfo.rawstart
@@ -38,12 +59,12 @@
hooks = Hooks()
null = lltype.nullptr(cintf.VMPROFSTACK)
- cintf.vmprof_tl_stack.setraw(null) # make it empty
- self.meta_interp(f, [10], policy=JitPolicy(hooks))
- v = set(visited)
- assert 0 in v
- v.remove(0)
- assert len(v) == 1
- assert 0 <= list(v)[0] - hooks.raw_start <= 10*1024
- assert cintf.vmprof_tl_stack.getraw() == null
+ self.meta_interp(main, [10], policy=JitPolicy(hooks))
+ print visited
+ #v = set(visited)
+ #assert 0 in v
+ #v.remove(0)
+ #assert len(v) == 1
+ #assert 0 <= list(v)[0] - hooks.raw_start <= 10*1024
+ #assert cintf.vmprof_tl_stack.getraw() == null
# ^^^ make sure we didn't leave anything dangling
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1673,25 +1673,6 @@
dest_addr = AddressLoc(base_loc, ofs_loc, scale, offset_loc.value)
self.save_into_mem(dest_addr, value_loc, size_loc)
- def genop_discard_strsetitem(self, op, arglocs):
- base_loc, ofs_loc, val_loc = arglocs
- basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR,
- self.cpu.translate_support_code)
- assert itemsize == 1
- dest_addr = AddressLoc(base_loc, ofs_loc, 0, basesize)
- self.mc.MOV8(dest_addr, val_loc.lowest8bits())
-
- def genop_discard_unicodesetitem(self, op, arglocs):
- base_loc, ofs_loc, val_loc = arglocs
- basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE,
- self.cpu.translate_support_code)
- if itemsize == 4:
- self.mc.MOV32(AddressLoc(base_loc, ofs_loc, 2, basesize), val_loc)
- elif itemsize == 2:
- self.mc.MOV16(AddressLoc(base_loc, ofs_loc, 1, basesize), val_loc)
- else:
- assert 0, itemsize
-
# genop_discard_setfield_raw = genop_discard_setfield_gc
def genop_math_read_timestamp(self, op, arglocs, resloc):
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -1219,6 +1219,7 @@
ofs_items, itemsize, _ = symbolic.get_array_token(rstr.STR,
self.translate_support_code)
assert itemsize == 1
+ ofs_items -= 1 # for the extra null character
scale = 0
self.assembler.load_effective_addr(ofsloc, ofs_items, scale,
resloc, baseloc)
diff --git a/rpython/jit/backend/x86/test/test_rvmprof.py b/rpython/jit/backend/x86/test/test_rvmprof.py
--- a/rpython/jit/backend/x86/test/test_rvmprof.py
+++ b/rpython/jit/backend/x86/test/test_rvmprof.py
@@ -3,5 +3,5 @@
from rpython.jit.backend.test.test_rvmprof import BaseRVMProfTest
from rpython.jit.backend.x86.test.test_basic import Jit386Mixin
-class TestFfiCall(Jit386Mixin, BaseRVMProfTest):
- pass
\ No newline at end of file
+class TestRVMProfCall(Jit386Mixin, BaseRVMProfTest):
+ pass
diff --git a/rpython/jit/backend/zarch/opassembler.py b/rpython/jit/backend/zarch/opassembler.py
--- a/rpython/jit/backend/zarch/opassembler.py
+++ b/rpython/jit/backend/zarch/opassembler.py
@@ -991,6 +991,7 @@
basesize, itemsize, _ = symbolic.get_array_token(rstr.STR,
self.cpu.translate_support_code)
assert itemsize == 1
More information about the pypy-commit
mailing list