[pypy-commit] pypy py3.5: merge default into py3.5
mattip
pypy.commits at gmail.com
Thu Nov 23 15:08:55 EST 2017
Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r93152:ce6402cbdf3c
Date: 2017-11-23 22:08 +0200
http://bitbucket.org/pypy/pypy/changeset/ce6402cbdf3c/
Log: merge default into py3.5
diff --git a/pypy/module/_io/interp_stringio.py b/pypy/module/_io/interp_stringio.py
--- a/pypy/module/_io/interp_stringio.py
+++ b/pypy/module/_io/interp_stringio.py
@@ -184,9 +184,7 @@
start,
end
)
- if endpos >= 0:
- endpos += start
- else:
+ if endpos < 0:
endpos = end
assert endpos >= 0
self.pos = endpos
diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -216,44 +216,41 @@
def _find_line_ending(self, line, start, end):
size = end - start
- if self.readtranslate:
-
- # Newlines are already translated, only search for \n
- pos = line.find(u'\n', start, end)
- if pos >= 0:
- return pos - start + 1, 0
- else:
- return -1, size
- elif self.readuniversal:
+ if self.readuniversal:
# Universal newline search. Find any of \r, \r\n, \n
# The decoder ensures that \r\n are not split in two pieces
- i = 0
+ i = start
while True:
- # Fast path for non-control chars. The loop always ends
- # since the Py_UNICODE storage is NUL-terminated.
- while i < size and line[start + i] > '\r':
+ # Fast path for non-control chars.
+ while i < end and line[i] > '\r':
i += 1
- if i >= size:
+ if i >= end:
return -1, size
- ch = line[start + i]
+ ch = line[i]
i += 1
if ch == '\n':
return i, 0
if ch == '\r':
- if line[start + i] == '\n':
+ if line[i] == '\n':
return i + 1, 0
else:
return i, 0
+ if self.readtranslate:
+ # Newlines are already translated, only search for \n
+ newline = u'\n'
else:
# Non-universal mode.
- pos = line.find(self.readnl, start, end)
- if pos >= 0:
- return pos - start + len(self.readnl), 0
- else:
- pos = line.find(self.readnl[0], start, end)
- if pos >= 0:
- return -1, pos - start
- return -1, size
+ newline = self.readnl
+ end_scan = end - len(newline) + 1
+ for i in range(start, end_scan):
+ ch = line[i]
+ if ch == newline[0]:
+ for j in range(1, len(newline)):
+ if line[i + j] != newline[j]:
+ break
+ else:
+ return i + len(newline), 0
+ return -1, end_scan
W_TextIOBase.typedef = TypeDef(
@@ -549,8 +546,13 @@
# _____________________________________________________________
# read methods
- def _set_decoded_chars(self, chars):
- self.decoded_chars = chars
+ def _unset_decoded(self):
+ self.decoded_chars = None
+ self.decoded_chars_used = 0
+
+ def _set_decoded(self, space, w_decoded):
+ check_decoded(space, w_decoded)
+ self.decoded_chars = space.unicode_w(w_decoded)
self.decoded_chars_used = 0
def _get_decoded_chars(self, size):
@@ -574,6 +576,10 @@
self.decoded_chars_used += size
return chars
+ def _has_data(self):
+ return (self.decoded_chars is not None and
+ self.decoded_chars_used < len(self.decoded_chars))
+
def _read_chunk(self, space):
"""Read and decode the next chunk of data from the BufferedReader.
The return value is True unless EOF was reached. The decoded string
@@ -616,8 +622,7 @@
eof = input_buf.getlength() == 0
w_decoded = space.call_method(self.w_decoder, "decode",
w_input, space.newbool(eof))
- check_decoded(space, w_decoded)
- self._set_decoded_chars(space.unicode_w(w_decoded))
+ self._set_decoded(space, w_decoded)
if space.len_w(w_decoded) > 0:
eof = False
@@ -629,6 +634,19 @@
return not eof
+ def _ensure_data(self, space):
+ while not self._has_data():
+ try:
+ if not self._read_chunk(space):
+ self._unset_decoded()
+ self.snapshot = None
+ return False
+ except OperationError as e:
+ if trap_eintr(space, e):
+ continue
+ raise
+ return True
+
def next_w(self, space):
self._check_attached(space)
self.telling = False
@@ -662,23 +680,13 @@
builder = UnicodeBuilder(size)
# Keep reading chunks until we have n characters to return
- while True:
+ while remaining > 0:
+ if not self._ensure_data(space):
+ break
data = self._get_decoded_chars(remaining)
builder.append(data)
remaining -= len(data)
- if remaining <= 0: # Done
- break
-
- try:
- if not self._read_chunk(space):
- # EOF
- break
- except OperationError as e:
- if trap_eintr(space, e):
- continue
- raise
-
return space.newunicode(builder.build())
def readline_w(self, space, w_limit=None):
@@ -687,28 +695,16 @@
self._writeflush(space)
limit = convert_size(space, w_limit)
- chunked = 0
line = None
remaining = None
- chunks = []
+ builder = UnicodeBuilder()
while True:
# First, get some data if necessary
- has_data = True
- while not self.decoded_chars:
- try:
- if not self._read_chunk(space):
- has_data = False
- break
- except OperationError as e:
- if trap_eintr(space, e):
- continue
- raise
+ has_data = self._ensure_data(space)
if not has_data:
# end of file
- self._set_decoded_chars(None)
- self.snapshot = None
start = endpos = offset_to_buffer = 0
break
@@ -725,8 +721,8 @@
line_len = len(line)
endpos, consumed = self._find_line_ending(line, start, line_len)
+ chunked = builder.getlength()
if endpos >= 0:
- endpos += start
if limit >= 0 and endpos >= start + limit - chunked:
endpos = start + limit - chunked
assert endpos >= 0
@@ -744,15 +740,15 @@
# No line ending seen yet - put aside current data
if endpos > start:
s = line[start:endpos]
- chunks.append(s)
- chunked += len(s)
+ builder.append(s)
+
# There may be some remaining bytes we'll have to prepend to the
# next chunk of data
if endpos < line_len:
remaining = line[endpos:]
line = None
# We have consumed the buffer
- self._set_decoded_chars(None)
+ self._unset_decoded()
if line:
# Our line ends in the current buffer
@@ -761,18 +757,12 @@
self.decoded_chars_used = decoded_chars_used
if start > 0 or endpos < len(line):
line = line[start:endpos]
- if remaining:
- chunks.append(remaining)
- remaining = None
- if chunks:
- if line:
- chunks.append(line)
- line = u''.join(chunks)
+ builder.append(line)
+ elif remaining:
+ builder.append(remaining)
- if line:
- return space.newunicode(line)
- else:
- return space.newunicode(u'')
+ result = builder.build()
+ return space.newunicode(result)
# _____________________________________________________________
# write methods
@@ -913,7 +903,7 @@
self._unsupportedoperation(
space, "can't do nonzero end-relative seeks")
space.call_method(self, "flush")
- self._set_decoded_chars(None)
+ self._unset_decoded()
self.snapshot = None
if self.w_decoder:
space.call_method(self.w_decoder, "reset")
@@ -943,7 +933,7 @@
# Seek back to the safe start point
space.call_method(self.w_buffer, "seek", space.newint(cookie.start_pos))
- self._set_decoded_chars(None)
+ self._unset_decoded()
self.snapshot = None
# Restore the decoder to its state from the safe start point.
@@ -964,8 +954,7 @@
w_decoded = space.call_method(self.w_decoder, "decode",
w_chunk, space.newbool(bool(cookie.need_eof)))
- check_decoded(space, w_decoded)
- self._set_decoded_chars(space.unicode_w(w_decoded))
+ self._set_decoded(space, w_decoded)
# Skip chars_to_skip of the decoded characters
if len(self.decoded_chars) < cookie.chars_to_skip:
@@ -1034,7 +1023,7 @@
w_decoded = space.call_method(self.w_decoder, "decode",
space.newbytes(input[i]))
check_decoded(space, w_decoded)
- chars_decoded += len(space.unicode_w(w_decoded))
+ chars_decoded += space.len_w(w_decoded)
cookie.bytes_to_feed += 1
diff --git a/requirements.txt b/requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,7 @@
cffi>=1.4.0
-vmprof>=0.4.10 # required to parse log files in rvmprof tests
+
+# parse log files in rvmprof tests
+vmprof>=0.4.10; 'x86' in platform.machine #skip arm, s390x
# hypothesis is used for test generation on untranslated tests
hypothesis
diff --git a/rpython/rlib/rstacklet.py b/rpython/rlib/rstacklet.py
--- a/rpython/rlib/rstacklet.py
+++ b/rpython/rlib/rstacklet.py
@@ -3,7 +3,6 @@
from rpython.rlib import jit
from rpython.rlib.objectmodel import fetch_translated_config
from rpython.rtyper.lltypesystem import lltype, llmemory
-from rpython.rlib import rvmprof
from rpython.rlib.rvmprof import cintf
DEBUG = False
@@ -41,13 +40,11 @@
def switch(self, stacklet):
if DEBUG:
debug.remove(stacklet)
- rvmprof.stop_sampling()
x = cintf.save_rvmprof_stack()
try:
h = self._gcrootfinder.switch(stacklet)
finally:
cintf.restore_rvmprof_stack(x)
- rvmprof.start_sampling()
if DEBUG:
debug.add(h)
return h
diff --git a/rpython/rlib/rvmprof/__init__.py b/rpython/rlib/rvmprof/__init__.py
--- a/rpython/rlib/rvmprof/__init__.py
+++ b/rpython/rlib/rvmprof/__init__.py
@@ -56,8 +56,10 @@
return None
def stop_sampling():
- fd = _get_vmprof().cintf.vmprof_stop_sampling()
+ from rpython.rlib.rvmprof.cintf import vmprof_stop_sampling
+ fd = vmprof_stop_sampling()
return rffi.cast(lltype.Signed, fd)
def start_sampling():
- _get_vmprof().cintf.vmprof_start_sampling()
+ from rpython.rlib.rvmprof.cintf import vmprof_start_sampling
+ vmprof_start_sampling()
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -9,6 +9,7 @@
from rpython.rtyper.tool import rffi_platform as platform
from rpython.rlib import rthread, jit
from rpython.rlib.objectmodel import we_are_translated
+from rpython.config.translationoption import get_translation_config
class VMProfPlatformUnsupported(Exception):
pass
@@ -40,7 +41,7 @@
compile_extra += ['-DVMPROF_UNIX']
compile_extra += ['-DVMPROF_LINUX']
elif sys.platform == 'win32':
- compile_extra = ['-DRPYTHON_VMPROF', '-DVMPROF_WINDOWS']
+ compile_extra += ['-DVMPROF_WINDOWS']
separate_module_files = [SHARED.join('vmprof_win.c')]
_libs = []
else:
@@ -120,16 +121,32 @@
vmprof_get_profile_path = rffi.llexternal("vmprof_get_profile_path", [rffi.CCHARP, lltype.Signed],
lltype.Signed, compilation_info=eci,
_nowrapper=True)
- vmprof_stop_sampling = rffi.llexternal("vmprof_stop_sampling", [],
- rffi.INT, compilation_info=eci,
- _nowrapper=True)
- vmprof_start_sampling = rffi.llexternal("vmprof_start_sampling", [],
- lltype.Void, compilation_info=eci,
- _nowrapper=True)
return CInterface(locals())
+# this is always present, but compiles to no-op if RPYTHON_VMPROF is not
+# defined (i.e. if we don't actually use vmprof in the generated C)
+auto_eci = ExternalCompilationInfo(post_include_bits=["""
+#ifndef RPYTHON_VMPROF
+# define vmprof_stop_sampling() (-1)
+# define vmprof_start_sampling() ((void)0)
+#endif
+"""])
+
+if get_translation_config() is None:
+ # tests need the full eci here
+ _eci = global_eci
+else:
+ _eci = auto_eci
+
+vmprof_stop_sampling = rffi.llexternal("vmprof_stop_sampling", [],
+ rffi.INT, compilation_info=_eci,
+ _nowrapper=True)
+vmprof_start_sampling = rffi.llexternal("vmprof_start_sampling", [],
+ lltype.Void, compilation_info=_eci,
+ _nowrapper=True)
+
class CInterface(object):
def __init__(self, namespace):
@@ -218,6 +235,7 @@
# stacklet support
def save_rvmprof_stack():
+ vmprof_stop_sampling()
return vmprof_tl_stack.get_or_make_raw()
def empty_rvmprof_stack():
@@ -225,6 +243,7 @@
def restore_rvmprof_stack(x):
vmprof_tl_stack.setraw(x)
+ vmprof_start_sampling()
#
# traceback support
diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c
--- a/rpython/rlib/rvmprof/src/rvmprof.c
+++ b/rpython/rlib/rvmprof/src/rvmprof.c
@@ -12,6 +12,7 @@
#endif
+#include "vmprof_common.h"
#include "shared/vmprof_get_custom_offset.h"
#ifdef VMPROF_UNIX
@@ -30,7 +31,7 @@
}
#endif
-long vmprof_get_profile_path(const char * buffer, long size)
+long vmprof_get_profile_path(char * buffer, long size)
{
return vmp_fd_to_path(vmp_profile_fileno(), buffer, size);
}
diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h
--- a/rpython/rlib/rvmprof/src/rvmprof.h
+++ b/rpython/rlib/rvmprof/src/rvmprof.h
@@ -36,8 +36,8 @@
RPY_EXTERN int vmprof_stack_append(void*, long);
RPY_EXTERN long vmprof_stack_pop(void*);
RPY_EXTERN void vmprof_stack_free(void*);
-RPY_EXTERN intptr_t vmprof_get_traceback(void *, void *, intptr_t*, intptr_t);
-RPY_EXTERN long vmprof_get_profile_path(const char *, long);
+RPY_EXTERN intptr_t vmprof_get_traceback(void *, void *, void**, intptr_t);
+RPY_EXTERN long vmprof_get_profile_path(char *, long);
RPY_EXTERN int vmprof_stop_sampling(void);
RPY_EXTERN void vmprof_start_sampling(void);
diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
--- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c
+++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
@@ -262,7 +262,7 @@
}
int depth = 0;
- PY_STACK_FRAME_T * top_most_frame = frame;
+ //PY_STACK_FRAME_T * top_most_frame = frame;
while ((depth + _per_loop()) <= max_depth) {
unw_get_proc_info(&cursor, &pip);
@@ -400,7 +400,7 @@
if (fd == NULL) {
return 0;
}
- char * saveptr;
+ char * saveptr = NULL;
char * line = NULL;
char * he = NULL;
char * name;
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_common.c b/rpython/rlib/rvmprof/src/shared/vmprof_common.c
--- a/rpython/rlib/rvmprof/src/shared/vmprof_common.c
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_common.c
@@ -4,6 +4,9 @@
#include <errno.h>
#ifdef RPYTHON_VMPROF
+
+int get_stack_trace(PY_THREAD_STATE_T * current, void** result, int max_depth, intptr_t pc);
+
#ifdef RPYTHON_LL2CTYPES
/* only for testing: ll2ctypes sets RPY_EXTERN from the command-line */
@@ -193,7 +196,7 @@
#endif
intptr_t vmprof_get_traceback(void *stack, void *ucontext,
- intptr_t *result_p, intptr_t result_length)
+ void **result_p, intptr_t result_length)
{
int n;
int enabled;
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_common.h b/rpython/rlib/rvmprof/src/shared/vmprof_common.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_common.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_common.h
@@ -96,7 +96,7 @@
#endif
RPY_EXTERN
intptr_t vmprof_get_traceback(void *stack, void *ucontext,
- intptr_t *result_p, intptr_t result_length);
+ void **result_p, intptr_t result_length);
#endif
int vmprof_get_signal_type(void);
diff --git a/rpython/rlib/rvmprof/test/test_rvmprof.py b/rpython/rlib/rvmprof/test/test_rvmprof.py
--- a/rpython/rlib/rvmprof/test/test_rvmprof.py
+++ b/rpython/rlib/rvmprof/test/test_rvmprof.py
@@ -144,7 +144,8 @@
@pytest.fixture
def init(self, tmpdir):
- eci = ExternalCompilationInfo(compile_extra=['-g','-O0'],
+ eci = ExternalCompilationInfo(compile_extra=['-g','-O0', '-Werror'],
+ post_include_bits = ['int native_func(int);'],
separate_module_sources=["""
RPY_EXTERN int native_func(int d) {
int j = 0;
More information about the pypy-commit
mailing list