[pypy-commit] pypy release-2.4.x: merge default into release
mattip
noreply at buildbot.pypy.org
Thu Sep 18 13:31:41 CEST 2014
Author: mattip <matti.picus at gmail.com>
Branch: release-2.4.x
Changeset: r73601:f5dcc2477b97
Date: 2014-09-18 14:28 +0300
http://bitbucket.org/pypy/pypy/changeset/f5dcc2477b97/
Log: merge default into release
diff --git a/lib-python/2.7/test/test_mmap.py b/lib-python/2.7/test/test_mmap.py
--- a/lib-python/2.7/test/test_mmap.py
+++ b/lib-python/2.7/test/test_mmap.py
@@ -179,25 +179,27 @@
import sys
f = open(TESTFN, "r+b")
try:
- m = mmap.mmap(f.fileno(), mapsize+1)
- except ValueError:
- # we do not expect a ValueError on Windows
- # CAUTION: This also changes the size of the file on disk, and
- # later tests assume that the length hasn't changed. We need to
- # repair that.
+ try:
+ m = mmap.mmap(f.fileno(), mapsize+1)
+ except ValueError:
+ # we do not expect a ValueError on Windows
+ # CAUTION: This also changes the size of the file on disk, and
+ # later tests assume that the length hasn't changed. We need to
+ # repair that.
+ if sys.platform.startswith('win'):
+ self.fail("Opening mmap with size+1 should work on Windows.")
+ else:
+ # we expect a ValueError on Unix, but not on Windows
+ if not sys.platform.startswith('win'):
+ self.fail("Opening mmap with size+1 should raise ValueError.")
+ m.close()
+ finally:
+ f.close()
if sys.platform.startswith('win'):
- self.fail("Opening mmap with size+1 should work on Windows.")
- else:
- # we expect a ValueError on Unix, but not on Windows
- if not sys.platform.startswith('win'):
- self.fail("Opening mmap with size+1 should raise ValueError.")
- m.close()
- f.close()
- if sys.platform.startswith('win'):
- # Repair damage from the resizing test.
- f = open(TESTFN, 'r+b')
- f.truncate(mapsize)
- f.close()
+ # Repair damage from the resizing test.
+ f = open(TESTFN, 'r+b')
+ f.truncate(mapsize)
+ f.close()
# Opening mmap with access=ACCESS_WRITE
f = open(TESTFN, "r+b")
diff --git a/pypy/doc/getting-started-python.rst b/pypy/doc/getting-started-python.rst
--- a/pypy/doc/getting-started-python.rst
+++ b/pypy/doc/getting-started-python.rst
@@ -111,6 +111,10 @@
of your choice. Typical example: ``--opt=2`` gives a good (but of
course slower) Python interpreter without the JIT.
+ Consider using PyPy instead of CPython in the above command line,
+ as it is much faster. (Note that ``rpython`` is a Python 2 program,
+ not Python 3; you need to run either PyPy 2 or CPython 2.)
+
.. _`optimization level`: config/opt.html
If everything works correctly this will create an executable
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
.. toctree::
+ release-2.4.0.rst
release-2.3.1.rst
release-2.3.0.rst
release-2.2.1.rst
diff --git a/pypy/doc/release-2.4.0.rst b/pypy/doc/release-2.4.0.rst
--- a/pypy/doc/release-2.4.0.rst
+++ b/pypy/doc/release-2.4.0.rst
@@ -1,11 +1,11 @@
=================================================
-PyPy 2.4 - ????????
+PyPy 2.4 - Snow White
=================================================
We're pleased to announce PyPy 2.4, which contains significant performance
enhancements and bug fixes.
-You can download the PyPy 2.4 release here:
+You can already download the PyPy 2.4-beta1 pre-release here:
http://pypy.org/download.html
@@ -58,7 +58,8 @@
bytearray handling, and a major rewrite of the GIL handling. This means
that external calls are now a lot faster, especially the CFFI ones. It also
means better performance in a lot of corner cases with handling strings or
-bytearrays.
+bytearrays. The main bugfix is handling of many socket objects in your
+program which in the long run used to "leak" memory.
PyPy now uses Python 2.7.8 standard library.
@@ -82,7 +83,7 @@
this mostly affects errno handling on linux, which makes external calls
faster.
-* Move to a mixed polling and mutex GIL model that make mutli-threaded jitted
+* Move to a mixed polling and mutex GIL model that make mutlithreaded jitted
code run *much* faster
* Optimize errno handling in linux (x86 and x86-64 only)
@@ -97,13 +98,17 @@
* Upgrade stdlib from 2.7.5 to 2.7.8
+* Win32 now links statically to zlib, expat, bzip, and openssl-1.0.1i.
+ No more missing DLLs
+
* Many issues were resolved_ since the 2.3.1 release on June 8
.. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html
.. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved
-We have further improvements on the way: rpython file handling and
-usable numpy linalg compatabiity should be merged soon.
+We have further improvements on the way: rpython file handling,
+numpy linalg compatibility, as well
+as improved GC and many smaller improvements.
Please try it out and let us know what you think. We especially welcome
success stories, we know you are using PyPy, please tell us about it!
diff --git a/pypy/doc/whatsnew-2.4.0.rst b/pypy/doc/whatsnew-2.4.0.rst
--- a/pypy/doc/whatsnew-2.4.0.rst
+++ b/pypy/doc/whatsnew-2.4.0.rst
@@ -60,3 +60,7 @@
.. branch: stdlib-2.7.8
Upgrades from 2.7.6 to 2.7.8
+
+.. branch: cpybug-seq-radd-rmul
+Fix issue #1861 - cpython compatability madness
+
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -4,5 +4,5 @@
=======================
.. this is a revision shortly after release-2.4.x
-.. startrev: 204b550079b0
+.. startrev: 7026746cbb1b
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -85,10 +85,13 @@
Abridged method (for -Ojit builds using Visual Studio 2008)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Download the versions of all the external packages
-from
+Download the versions of all the external packages from
+https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip
+(for 2.4 release and later) or
https://bitbucket.org/pypy/pypy/downloads/local.zip
-Then expand it into the base directory (base_dir) and modify your environment to reflect this::
+(for pre-2.4 versions)
+Then expand it into the base directory (base_dir) and modify your environment
+to reflect this::
set PATH=<base_dir>\bin;<base_dir>\tcltk\bin;%PATH%
set INCLUDE=<base_dir>\include;<base_dir>\tcltk\include;%INCLUDE%
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -38,18 +38,15 @@
def cpython_code_signature(code):
"([list-of-arg-names], vararg-name-or-None, kwarg-name-or-None)."
argcount = code.co_argcount
+ varnames = code.co_varnames
assert argcount >= 0 # annotator hint
- argnames = list(code.co_varnames[:argcount])
+ argnames = list(varnames[:argcount])
if code.co_flags & CO_VARARGS:
- varargname = code.co_varnames[argcount]
+ varargname = varnames[argcount]
argcount += 1
else:
varargname = None
- if code.co_flags & CO_VARKEYWORDS:
- kwargname = code.co_varnames[argcount]
- argcount += 1
- else:
- kwargname = None
+ kwargname = varnames[argcount] if code.co_flags & CO_VARKEYWORDS else None
return Signature(argnames, varargname, kwargname)
diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -83,12 +83,6 @@
v = PyString_DecodeEscape(space, substr, 'strict', enc)
return space.wrap(v)
-def hexbyte(val):
- result = "%x" % val
- if len(result) == 1:
- result = "0" + result
- return result
-
def decode_unicode_utf8(space, s, ps, q):
# ****The Python 2.7 version, producing UTF-32 escapes****
# String is utf8-encoded, but 'unicode_escape' expects
@@ -108,15 +102,14 @@
# instead.
lis.append("u005c")
if ord(s[ps]) & 0x80: # XXX inefficient
- w, ps = decode_utf8(space, s, ps, end, "utf-32-be")
- rn = len(w)
- assert rn % 4 == 0
- for i in range(0, rn, 4):
- lis.append('\\U')
- lis.append(hexbyte(ord(w[i])))
- lis.append(hexbyte(ord(w[i+1])))
- lis.append(hexbyte(ord(w[i+2])))
- lis.append(hexbyte(ord(w[i+3])))
+ w, ps = decode_utf8(space, s, ps, end)
+ for c in w:
+ # The equivalent of %08x, which is not supported by RPython.
+ # 7 zeroes are enough for the unicode range, and the
+ # result still fits in 32-bit.
+ hexa = hex(ord(c) + 0x10000000)
+ lis.append('\\U0')
+ lis.append(hexa[3:]) # Skip 0x and the leading 1
else:
lis.append(s[ps])
ps += 1
@@ -136,7 +129,7 @@
# note that the C code has a label here.
# the logic is the same.
if recode_encoding and ord(s[ps]) & 0x80:
- w, ps = decode_utf8(space, s, ps, end, recode_encoding)
+ w, ps = decode_utf8_recode(space, s, ps, end, recode_encoding)
# Append bytes to output buffer.
builder.append(w)
else:
@@ -222,14 +215,18 @@
ch >= 'A' and ch <= 'F')
-def decode_utf8(space, s, ps, end, encoding):
+def decode_utf8(space, s, ps, end):
assert ps >= 0
pt = ps
# while (s < end && *s != '\\') s++; */ /* inefficient for u".."
while ps < end and ord(s[ps]) & 0x80:
ps += 1
- w_u = space.wrap(unicodehelper.decode_utf8(space, s[pt:ps]))
- w_v = unicodehelper.encode(space, w_u, encoding)
+ u = unicodehelper.decode_utf8(space, s[pt:ps])
+ return u, ps
+
+def decode_utf8_recode(space, s, ps, end, recode_encoding):
+ u, ps = decode_utf8(space, s, ps, end)
+ w_v = unicodehelper.encode(space, space.wrap(u), recode_encoding)
v = space.str_w(w_v)
return v, ps
diff --git a/pypy/interpreter/pyparser/test/test_parsestring.py b/pypy/interpreter/pyparser/test/test_parsestring.py
--- a/pypy/interpreter/pyparser/test/test_parsestring.py
+++ b/pypy/interpreter/pyparser/test/test_parsestring.py
@@ -73,11 +73,11 @@
def test_simple_enc_roundtrip(self):
space = self.space
- s = "'\x81'"
+ s = "'\x81\\t'"
s = s.decode("koi8-u").encode("utf8")
w_ret = parsestring.parsestr(self.space, 'koi8-u', s)
ret = space.unwrap(w_ret)
- assert ret == eval("# -*- coding: koi8-u -*-\n'\x81'")
+ assert ret == eval("# -*- coding: koi8-u -*-\n'\x81\\t'")
def test_multiline_unicode_strings_with_backslash(self):
space = self.space
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -945,7 +945,7 @@
prefix = udir.join('pathtest').ensure(dir=1)
fake_exe = 'bin/pypy-c'
if sys.platform == 'win32':
- fake_exe += '.exe'
+ fake_exe = 'pypy-c.exe'
fake_exe = prefix.join(fake_exe).ensure(file=1)
expected_path = [str(prefix.join(subdir).ensure(dir=1))
for subdir in ('lib_pypy',
@@ -985,8 +985,10 @@
assert sys.path == old_sys_path + [self.goal_dir]
app_main.setup_bootstrap_path(self.fake_exe)
- assert sys.executable == '' # not executable!
- assert sys.path == old_sys_path + [self.goal_dir]
+ if not sys.platform == 'win32':
+ # an existing file is always 'executable' on windows
+ assert sys.executable == '' # not executable!
+ assert sys.path == old_sys_path + [self.goal_dir]
os.chmod(self.fake_exe, 0755)
app_main.setup_bootstrap_path(self.fake_exe)
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -5,6 +5,7 @@
@specialize.memo()
def decode_error_handler(space):
+ # Fast version of the "strict" errors handler.
def raise_unicode_exception_decode(errors, encoding, msg, s,
startingpos, endingpos):
raise OperationError(space.w_UnicodeDecodeError,
@@ -17,6 +18,7 @@
@specialize.memo()
def encode_error_handler(space):
+ # Fast version of the "strict" errors handler.
def raise_unicode_exception_encode(errors, encoding, msg, u,
startingpos, endingpos):
raise OperationError(space.w_UnicodeEncodeError,
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -8,6 +8,7 @@
from rpython.rlib.jit_libffi import (CIF_DESCRIPTION, CIF_DESCRIPTION_P,
FFI_TYPE, FFI_TYPE_P, FFI_TYPE_PP, SIZE_OF_FFI_ARG)
from rpython.rlib.objectmodel import we_are_translated, instantiate
+from rpython.rlib.objectmodel import keepalive_until_here
from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
from pypy.interpreter.error import OperationError, oefmt
@@ -160,6 +161,7 @@
raw_cdata = rffi.cast(rffi.CCHARPP, data)[0]
lltype.free(raw_cdata, flavor='raw')
lltype.free(buffer, flavor='raw')
+ keepalive_until_here(args_w)
return w_res
def get_mustfree_flag(data):
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -759,17 +759,25 @@
# socket's timeout is in seconds, poll's timeout in ms
timeout = int(sock_timeout * 1000 + 0.5)
- ready = rpoll.poll(fddict, timeout)
+ try:
+ ready = rpoll.poll(fddict, timeout)
+ except rpoll.PollError, e:
+ message = e.get_msg()
+ raise ssl_error(space, message, e.errno)
else:
if MAX_FD_SIZE is not None and sock_fd >= MAX_FD_SIZE:
return SOCKET_TOO_LARGE_FOR_SELECT
- if writing:
- r, w, e = rpoll.select([], [sock_fd], [], sock_timeout)
- ready = w
- else:
- r, w, e = rpoll.select([sock_fd], [], [], sock_timeout)
- ready = r
+ try:
+ if writing:
+ r, w, e = rpoll.select([], [sock_fd], [], sock_timeout)
+ ready = w
+ else:
+ r, w, e = rpoll.select([sock_fd], [], [], sock_timeout)
+ ready = r
+ except rpoll.SelectError as e:
+ message = e.get_msg()
+ raise ssl_error(space, message, e.errno)
if ready:
return SOCKET_OPERATION_OK
else:
diff --git a/pypy/module/operator/__init__.py b/pypy/module/operator/__init__.py
--- a/pypy/module/operator/__init__.py
+++ b/pypy/module/operator/__init__.py
@@ -39,7 +39,7 @@
'irshift', 'isub', 'itruediv', 'ixor', '_length_hint']
interpleveldefs = {
- '_compare_digest': 'interp_operator.compare_digest',
+ '_compare_digest': 'tscmp.compare_digest',
}
for name in interp_names:
diff --git a/pypy/module/operator/app_operator.py b/pypy/module/operator/app_operator.py
--- a/pypy/module/operator/app_operator.py
+++ b/pypy/module/operator/app_operator.py
@@ -4,7 +4,7 @@
This module exports a set of operators as functions. E.g. operator.add(x,y) is
equivalent to x+y.
'''
-from __pypy__ import builtinify
+
import types
@@ -27,7 +27,7 @@
'getslice(a, b, c) -- Same as a[b:c].'
if not isinstance(start, int) or not isinstance(end, int):
raise TypeError("an integer is expected")
- return a[start:end]
+ return a[start:end]
__getslice__ = getslice
def indexOf(a, b):
@@ -37,7 +37,7 @@
if x == b:
return index
index += 1
- raise ValueError, 'sequence.index(x): x not in sequence'
+ raise ValueError('sequence.index(x): x not in sequence')
def isMappingType(obj,):
'isMappingType(a) -- Return True if a has a mapping type, False otherwise.'
@@ -58,9 +58,9 @@
def repeat(obj, num):
'repeat(a, b) -- Return a * b, where a is a sequence, and b is an integer.'
if not isinstance(num, (int, long)):
- raise TypeError, 'an integer is required'
+ raise TypeError('an integer is required')
if not isSequenceType(obj):
- raise TypeError, "non-sequence object can't be repeated"
+ raise TypeError("non-sequence object can't be repeated")
return obj * num
@@ -68,59 +68,85 @@
def setslice(a, b, c, d):
'setslice(a, b, c, d) -- Same as a[b:c] = d.'
- a[b:c] = d
+ a[b:c] = d
__setslice__ = setslice
+def _resolve_attr_chain(chain, obj, idx=0):
+ obj = getattr(obj, chain[idx])
+ if idx + 1 == len(chain):
+ return obj
+ else:
+ return _resolve_attr_chain(chain, obj, idx + 1)
+
+
+class _simple_attrgetter(object):
+ def __init__(self, attr):
+ self._attr = attr
+
+ def __call__(self, obj):
+ return getattr(obj, self._attr)
+
+
+class _single_attrgetter(object):
+ def __init__(self, attrs):
+ self._attrs = attrs
+
+ def __call__(self, obj):
+ return _resolve_attr_chain(self._attrs, obj)
+
+
+class _multi_attrgetter(object):
+ def __init__(self, attrs):
+ self._attrs = attrs
+
+ def __call__(self, obj):
+ return tuple([
+ _resolve_attr_chain(attrs, obj)
+ for attrs in self._attrs
+ ])
+
+
def attrgetter(attr, *attrs):
+ if (
+ not isinstance(attr, basestring) or
+ not all(isinstance(a, basestring) for a in attrs)
+ ):
+ def _raise_typeerror(obj):
+ raise TypeError(
+ "argument must be a string, not %r" % type(attr).__name__
+ )
+ return _raise_typeerror
if attrs:
- getters = [single_attr_getter(a) for a in (attr,) + attrs]
- def getter(obj):
- return tuple([getter(obj) for getter in getters])
+ return _multi_attrgetter([
+ a.split(".") for a in [attr] + list(attrs)
+ ])
+ elif "." not in attr:
+ return _simple_attrgetter(attr)
else:
- getter = single_attr_getter(attr)
- return builtinify(getter)
+ return _single_attrgetter(attr.split("."))
-def single_attr_getter(attr):
- if not isinstance(attr, str):
- if not isinstance(attr, unicode):
- def _raise_typeerror(obj):
- raise TypeError("argument must be a string, not %r" %
- (type(attr).__name__,))
- return _raise_typeerror
- attr = attr.encode('ascii')
- #
- def make_getter(name, prevfn=None):
- if prevfn is None:
- def getter(obj):
- return getattr(obj, name)
+
+class itemgetter(object):
+ def __init__(self, item, *items):
+ self._single = not bool(items)
+ if self._single:
+ self._idx = item
else:
- def getter(obj):
- return getattr(prevfn(obj), name)
- return getter
- #
- last = 0
- getter = None
- while True:
- dot = attr.find(".", last)
- if dot < 0: break
- getter = make_getter(attr[last:dot], getter)
- last = dot + 1
- return make_getter(attr[last:], getter)
+ self._idx = [item] + list(items)
+ def __call__(self, obj):
+ if self._single:
+ return obj[self._idx]
+ else:
+ return tuple([obj[i] for i in self._idx])
-def itemgetter(item, *items):
- if items:
- list_of_indices = [item] + list(items)
- def getter(obj):
- return tuple([obj[i] for i in list_of_indices])
- else:
- def getter(obj):
- return obj[item]
- return builtinify(getter)
+class methodcaller(object):
+ def __init__(self, method_name, *args, **kwargs):
+ self._method_name = method_name
+ self._args = args
+ self._kwargs = kwargs
-def methodcaller(method_name, *args, **kwargs):
- def call(obj):
- return getattr(obj, method_name)(*args, **kwargs)
- return builtinify(call)
+ def __call__(self, obj):
+ return getattr(obj, self._method_name)(*self._args, **self._kwargs)
diff --git a/pypy/module/operator/interp_operator.py b/pypy/module/operator/interp_operator.py
--- a/pypy/module/operator/interp_operator.py
+++ b/pypy/module/operator/interp_operator.py
@@ -1,6 +1,4 @@
-from rpython.rlib.objectmodel import specialize
-
-from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.error import OperationError
from pypy.interpreter.gateway import unwrap_spec
@@ -249,33 +247,3 @@
@unwrap_spec(default=int)
def _length_hint(space, w_iterable, default):
return space.wrap(space.length_hint(w_iterable, default))
-
-def compare_digest(space, w_a, w_b):
- if (
- space.isinstance_w(w_a, space.w_unicode) and
- space.isinstance_w(w_b, space.w_unicode)
- ):
- return space.wrap(tscmp(space.unicode_w(w_a), space.unicode_w(w_b)))
- if (
- space.isinstance_w(w_a, space.w_unicode) or
- space.isinstance_w(w_b, space.w_unicode)
- ):
- raise oefmt(
- space.w_TypeError,
- "unsupported operand types(s) or combination of types: '%N' and '%N'",
- w_a,
- w_b,
- )
- else:
- return space.wrap(tscmp(space.bufferstr_w(w_a), space.bufferstr_w(w_b)))
-
-
- at specialize.argtype(0, 1)
-def tscmp(a, b):
- len_a = len(a)
- len_b = len(b)
- length = min(len(a), len(b))
- res = len_a ^ len_b
- for i in xrange(length):
- res |= ord(a[i]) ^ ord(b[i])
- return res == 0
diff --git a/pypy/module/operator/test/test_operator.py b/pypy/module/operator/test/test_operator.py
--- a/pypy/module/operator/test/test_operator.py
+++ b/pypy/module/operator/test/test_operator.py
@@ -334,3 +334,9 @@
assert operator._compare_digest(a, b)
a, b = mybytes(b"foobar"), mybytes(b"foobaz")
assert not operator._compare_digest(a, b)
+
+ def test_compare_digest_unicode(self):
+ import operator
+ assert operator._compare_digest(u'asd', u'asd')
+ assert not operator._compare_digest(u'asd', u'qwe')
+ raises(TypeError, operator._compare_digest, u'asd', b'qwe')
diff --git a/pypy/module/operator/test/test_tscmp.py b/pypy/module/operator/test/test_tscmp.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/operator/test/test_tscmp.py
@@ -0,0 +1,28 @@
+from pypy.module.operator.tscmp import pypy_tscmp, pypy_tscmp_wide
+
+class TestTimingSafeCompare:
+ tostr = str
+ tscmp = staticmethod(pypy_tscmp)
+
+ def test_tscmp_neq(self):
+ assert not self.tscmp(self.tostr('asd'), self.tostr('qwe'), 3, 3)
+
+ def test_tscmp_eq(self):
+ assert self.tscmp(self.tostr('asd'), self.tostr('asd'), 3, 3)
+
+ def test_tscmp_len(self):
+ assert self.tscmp(self.tostr('asdp'), self.tostr('asdq'), 3, 3)
+
+ def test_tscmp_nlen(self):
+ assert not self.tscmp(self.tostr('asd'), self.tostr('asd'), 2, 3)
+
+
+class TestTimingSafeCompareWide(TestTimingSafeCompare):
+ tostr = unicode
+ tscmp = staticmethod(pypy_tscmp_wide)
+
+ def test_tscmp_wide_nonascii(self):
+ a, b = u"\ud808\udf45", u"\ud808\udf45"
+ assert self.tscmp(a, b, len(a), len(b))
+ a, b = u"\ud808\udf45", u"\ud808\udf45 "
+ assert not self.tscmp(a, b, len(a), len(b))
diff --git a/pypy/module/operator/tscmp.c b/pypy/module/operator/tscmp.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/operator/tscmp.c
@@ -0,0 +1,80 @@
+/* Derived from CPython 3.3.5's operator.c::_tscmp
+ */
+
+#include <stdlib.h>
+#include <wchar.h>
+#include "tscmp.h"
+
+int
+pypy_tscmp(const char *a, const char *b, long len_a, long len_b)
+{
+ /* The volatile type declarations make sure that the compiler has no
+ * chance to optimize and fold the code in any way that may change
+ * the timing.
+ */
+ volatile long length;
+ volatile const char *left;
+ volatile const char *right;
+ long i;
+ char result;
+
+ /* loop count depends on length of b */
+ length = len_b;
+ left = NULL;
+ right = b;
+
+ /* don't use else here to keep the amount of CPU instructions constant,
+ * volatile forces re-evaluation
+ * */
+ if (len_a == length) {
+ left = *((volatile const char**)&a);
+ result = 0;
+ }
+ if (len_a != length) {
+ left = b;
+ result = 1;
+ }
+
+ for (i=0; i < length; i++) {
+ result |= *left++ ^ *right++;
+ }
+
+ return (result == 0);
+}
+
+int
+pypy_tscmp_wide(const wchar_t *a, const wchar_t *b, long len_a, long len_b)
+{
+ /* The volatile type declarations make sure that the compiler has no
+ * chance to optimize and fold the code in any way that may change
+ * the timing.
+ */
+ volatile long length;
+ volatile const wchar_t *left;
+ volatile const wchar_t *right;
+ long i;
+ wchar_t result;
+
+ /* loop count depends on length of b */
+ length = len_b;
+ left = NULL;
+ right = b;
+
+ /* don't use else here to keep the amount of CPU instructions constant,
+ * volatile forces re-evaluation
+ * */
+ if (len_a == length) {
+ left = *((volatile const wchar_t**)&a);
+ result = 0;
+ }
+ if (len_a != length) {
+ left = b;
+ result = 1;
+ }
+
+ for (i=0; i < length; i++) {
+ result |= *left++ ^ *right++;
+ }
+
+ return (result == 0);
+}
diff --git a/pypy/module/operator/tscmp.h b/pypy/module/operator/tscmp.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/operator/tscmp.h
@@ -0,0 +1,2 @@
+int pypy_tscmp(const char *, const char *, long, long);
+int pypy_tscmp_wide(const wchar_t *, const wchar_t *, long, long);
diff --git a/pypy/module/operator/tscmp.py b/pypy/module/operator/tscmp.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/operator/tscmp.py
@@ -0,0 +1,73 @@
+"""
+Provides _compare_digest method, which is a safe comparing to prevent timing
+attacks for the hmac module.
+"""
+import py
+
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+
+from pypy.interpreter.error import oefmt
+
+cwd = py.path.local(__file__).dirpath()
+eci = ExternalCompilationInfo(
+ includes=[cwd.join('tscmp.h')],
+ include_dirs=[str(cwd)],
+ separate_module_files=[cwd.join('tscmp.c')],
+ export_symbols=['pypy_tscmp', 'pypy_tscmp_wide'])
+
+
+def llexternal(*args, **kwargs):
+ kwargs.setdefault('compilation_info', eci)
+ kwargs.setdefault('sandboxsafe', True)
+ return rffi.llexternal(*args, **kwargs)
+
+
+pypy_tscmp = llexternal(
+ 'pypy_tscmp',
+ [rffi.CCHARP, rffi.CCHARP, rffi.LONG, rffi.LONG],
+ rffi.INT)
+pypy_tscmp_wide = llexternal(
+ 'pypy_tscmp_wide',
+ [rffi.CWCHARP, rffi.CWCHARP, rffi.LONG, rffi.LONG],
+ rffi.INT)
+
+
+def compare_digest(space, w_a, w_b):
+ """compare_digest(a, b) -> bool
+
+ Return 'a == b'. This function uses an approach designed to prevent
+ timing analysis, making it appropriate for cryptography. a and b
+ must both be of the same type: either str (ASCII only), or any type
+ that supports the buffer protocol (e.g. bytes).
+
+ Note: If a and b are of different lengths, or if an error occurs, a
+ timing attack could theoretically reveal information about the types
+ and lengths of a and b--but not their values.
+ """
+ if (space.isinstance_w(w_a, space.w_unicode) and
+ space.isinstance_w(w_b, space.w_unicode)):
+ a = space.unicode_w(w_a)
+ b = space.unicode_w(w_b)
+ with rffi.scoped_nonmoving_unicodebuffer(a) as a_buf:
+ with rffi.scoped_nonmoving_unicodebuffer(b) as b_buf:
+ result = pypy_tscmp_wide(a_buf, b_buf, len(a), len(b))
+ return space.wrap(rffi.cast(lltype.Bool, result))
+ return compare_digest_buffer(space, w_a, w_b)
+
+
+def compare_digest_buffer(space, w_a, w_b):
+ try:
+ a_buf = w_a.buffer_w(space, space.BUF_SIMPLE)
+ b_buf = w_b.buffer_w(space, space.BUF_SIMPLE)
+ except TypeError:
+ raise oefmt(space.w_TypeError,
+ "unsupported operand types(s) or combination of types: "
+ "'%T' and '%T'", w_a, w_b)
+
+ a = a_buf.as_str()
+ b = b_buf.as_str()
+ with rffi.scoped_nonmovingbuffer(a) as a_buf:
+ with rffi.scoped_nonmovingbuffer(b) as b_buf:
+ result = pypy_tscmp(a_buf, b_buf, len(a), len(b))
+ return space.wrap(rffi.cast(lltype.Bool, result))
diff --git a/pypy/module/sys/state.py b/pypy/module/sys/state.py
--- a/pypy/module/sys/state.py
+++ b/pypy/module/sys/state.py
@@ -7,15 +7,15 @@
# ____________________________________________________________
#
-class State:
- def __init__(self, space):
- self.space = space
+class State:
+ def __init__(self, space):
+ self.space = space
self.w_modules = space.newdict(module=True)
-
self.w_warnoptions = space.newlist([])
self.w_argv = space.newlist([])
- self.setinitialpath(space)
+
+ self.setinitialpath(space)
def setinitialpath(self, space):
from pypy.module.sys.initpath import compute_stdlib_path
@@ -25,10 +25,10 @@
path = compute_stdlib_path(self, srcdir)
self.w_path = space.newlist([space.wrap(p) for p in path])
-
def get(space):
return space.fromcache(State)
+
class IOState:
def __init__(self, space):
from pypy.module._file.interp_file import W_File
@@ -36,17 +36,17 @@
stdin = W_File(space)
stdin.file_fdopen(0, "r", 1)
- stdin.name = '<stdin>'
+ stdin.w_name = space.wrap('<stdin>')
self.w_stdin = space.wrap(stdin)
stdout = W_File(space)
stdout.file_fdopen(1, "w", 1)
- stdout.name = '<stdout>'
+ stdout.w_name = space.wrap('<stdout>')
self.w_stdout = space.wrap(stdout)
stderr = W_File(space)
stderr.file_fdopen(2, "w", 0)
- stderr.name = '<stderr>'
+ stderr.w_name = space.wrap('<stderr>')
self.w_stderr = space.wrap(stderr)
stdin._when_reading_first_flush(stdout)
@@ -54,9 +54,9 @@
def getio(space):
return space.fromcache(IOState)
+
def pypy_getudir(space):
"""NOT_RPYTHON
(should be removed from interpleveldefs before translation)"""
from rpython.tool.udir import udir
return space.wrap(str(udir))
-
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -91,6 +91,10 @@
assert isinstance(sys.__stderr__, file)
assert isinstance(sys.__stdin__, file)
+ #assert sys.__stdin__.name == "<stdin>"
+ #assert sys.__stdout__.name == "<stdout>"
+ #assert sys.__stderr__.name == "<stderr>"
+
if self.appdirect and not isinstance(sys.stdin, file):
return
diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py
--- a/rpython/annotator/builtin.py
+++ b/rpython/annotator/builtin.py
@@ -255,6 +255,11 @@
BUILTIN_ANALYZERS[original] = value
+ at analyzer_for(getattr(object.__init__, 'im_func', object.__init__))
+def object_init(s_self, *args):
+ # ignore - mostly used for abstract classes initialization
+ pass
+
@analyzer_for(getattr(EnvironmentError.__init__, 'im_func', EnvironmentError.__init__))
def EnvironmentError_init(s_self, *args):
pass
@@ -268,11 +273,6 @@
def WindowsError_init(s_self, *args):
pass
- at analyzer_for(getattr(object.__init__, 'im_func', object.__init__))
-def object_init(s_self, *args):
- # ignore - mostly used for abstract classes initialization
- pass
-
@analyzer_for(sys.getdefaultencoding)
def conf():
diff --git a/rpython/annotator/classdef.py b/rpython/annotator/classdef.py
--- a/rpython/annotator/classdef.py
+++ b/rpython/annotator/classdef.py
@@ -438,7 +438,9 @@
# ____________________________________________________________
FORCE_ATTRIBUTES_INTO_CLASSES = {
- EnvironmentError: {'errno': SomeInteger(), 'strerror': SomeString(can_be_None=True), 'filename': SomeString(can_be_None=True)},
+ EnvironmentError: {'errno': SomeInteger(),
+ 'strerror': SomeString(can_be_None=True),
+ 'filename': SomeString(can_be_None=True)},
}
try:
@@ -455,4 +457,3 @@
else:
FORCE_ATTRIBUTES_INTO_CLASSES[termios.error] = \
{'args': SomeTuple([SomeInteger(), SomeString()])}
-
diff --git a/rpython/jit/backend/arm/callbuilder.py b/rpython/jit/backend/arm/callbuilder.py
--- a/rpython/jit/backend/arm/callbuilder.py
+++ b/rpython/jit/backend/arm/callbuilder.py
@@ -92,7 +92,8 @@
self.mc.LDR_ri(r.r7.value, r.r5.value)
# change 'rpy_fastgil' to 0 (it should be non-zero right now)
- self.mc.DMB()
+ if self.asm.cpu.cpuinfo.arch_version >= 7:
+ self.mc.DMB()
self.mc.gen_load_int(r.r6.value, fastgil)
self.mc.MOV_ri(r.ip.value, 0)
self.mc.STR_ri(r.ip.value, r.r6.value)
@@ -112,7 +113,8 @@
self.mc.STREX(r.r3.value, r.ip.value, r.r6.value, c=c.EQ)
# try to claim the lock
self.mc.CMP_ri(r.r3.value, 0, cond=c.EQ) # did this succeed?
- self.mc.DMB()
+ if self.asm.cpu.cpuinfo.arch_version >= 7:
+ self.mc.DMB()
# the success of the lock acquisition is defined by
# 'EQ is true', or equivalently by 'r3 == 0'.
#
diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py
--- a/rpython/jit/backend/arm/codebuilder.py
+++ b/rpython/jit/backend/arm/codebuilder.py
@@ -333,6 +333,8 @@
| (rn & 0xF) << 16)
def DMB(self):
+ # ARMv7 only. I guess ARMv6 CPUs cannot be used in symmetric
+ # multi-processing at all? That would make this instruction unneeded.
# note: 'cond' is only permitted on Thumb here, but don't
# write literally 0xf57ff05f, because it's larger than 31 bits
c = cond.AL
diff --git a/rpython/jit/backend/arm/detect.py b/rpython/jit/backend/arm/detect.py
--- a/rpython/jit/backend/arm/detect.py
+++ b/rpython/jit/backend/arm/detect.py
@@ -38,9 +38,9 @@
try:
buf = os.read(fd, 2048)
if not buf:
+ n = 6 # we assume ARMv6 as base case
debug_print("Could not detect ARM architecture "
"version, assuming", "ARMv%d" % n)
- n = 6 # we asume ARMv6 as base case
finally:
os.close(fd)
# "Processor : ARMv%d-compatible processor rev 7 (v6l)"
diff --git a/rpython/jit/backend/arm/instructions.py b/rpython/jit/backend/arm/instructions.py
--- a/rpython/jit/backend/arm/instructions.py
+++ b/rpython/jit/backend/arm/instructions.py
@@ -142,6 +142,7 @@
#'VCVT' : {'opc1':0xB, 'opc2':0xE, 'opc3':0x1, 'base': False},
}
+# ARMv7 only
simd_instructions_3regs = {
'VADD_i64': {'A': 0x8, 'B': 0, 'U': 0},
'VSUB_i64': {'A': 0x8, 'B': 0, 'U': 1},
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -1128,6 +1128,8 @@
self.mc.VCVT_int_to_float(res.value, r.svfp_ip.value)
return fcond
+ # the following five instructions are only ARMv7;
+ # regalloc.py won't call them at all on ARMv6
emit_op_llong_add = gen_emit_float_op('llong_add', 'VADD_i64')
emit_op_llong_sub = gen_emit_float_op('llong_sub', 'VSUB_i64')
emit_op_llong_and = gen_emit_float_op('llong_and', 'VAND_i64')
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -184,7 +184,7 @@
class Regalloc(BaseRegalloc):
- def __init__(self, assembler=None):
+ def __init__(self, assembler):
self.cpu = assembler.cpu
self.assembler = assembler
self.frame_manager = None
@@ -290,7 +290,7 @@
return self.vfprm.convert_to_imm(value)
def _prepare(self, inputargs, operations, allgcrefs):
- cpu = self.assembler.cpu
+ cpu = self.cpu
self.fm = ARMFrameManager(cpu.get_baseofs_of_frame_field())
self.frame_manager = self.fm
operations = cpu.gc_ll_descr.rewrite_assembler(cpu, operations,
@@ -550,18 +550,19 @@
EffectInfo.OS_LLONG_AND,
EffectInfo.OS_LLONG_OR,
EffectInfo.OS_LLONG_XOR):
- args = self._prepare_llong_binop_xx(op, fcond)
- self.perform_llong(op, args, fcond)
- return
- if oopspecindex == EffectInfo.OS_LLONG_TO_INT:
+ if self.cpu.cpuinfo.arch_version >= 7:
+ args = self._prepare_llong_binop_xx(op, fcond)
+ self.perform_llong(op, args, fcond)
+ return
+ elif oopspecindex == EffectInfo.OS_LLONG_TO_INT:
args = self._prepare_llong_to_int(op, fcond)
self.perform_llong(op, args, fcond)
return
- if oopspecindex == EffectInfo.OS_MATH_SQRT:
+ elif oopspecindex == EffectInfo.OS_MATH_SQRT:
args = self.prepare_op_math_sqrt(op, fcond)
self.perform_math(op, args, fcond)
return
- #if oopspecindex == EffectInfo.OS_MATH_READ_TIMESTAMP:
+ #elif oopspecindex == EffectInfo.OS_MATH_READ_TIMESTAMP:
# ...
return self._prepare_call(op)
@@ -590,7 +591,7 @@
# spill variables that need to be saved around calls
self.vfprm.before_call(save_all_regs=save_all_regs)
if not save_all_regs:
- gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap
+ gcrootmap = self.cpu.gc_ll_descr.gcrootmap
if gcrootmap and gcrootmap.is_shadow_stack:
save_all_regs = 2
self.rm.before_call(save_all_regs=save_all_regs)
@@ -1082,7 +1083,7 @@
gcmap = self.get_gcmap([r.r0, r.r1])
self.possibly_free_var(t)
#
- gc_ll_descr = self.assembler.cpu.gc_ll_descr
+ gc_ll_descr = self.cpu.gc_ll_descr
self.assembler.malloc_cond_varsize_frame(
gc_ll_descr.get_nursery_free_addr(),
gc_ll_descr.get_nursery_top_addr(),
@@ -1092,7 +1093,7 @@
self.assembler._alignment_check()
def prepare_op_call_malloc_nursery_varsize(self, op, fcond):
- gc_ll_descr = self.assembler.cpu.gc_ll_descr
+ gc_ll_descr = self.cpu.gc_ll_descr
if not hasattr(gc_ll_descr, 'max_size_of_young_obj'):
raise Exception("unreachable code")
# for boehm, this function should never be called
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -1,3 +1,4 @@
+from rpython.rlib import rgc
from rpython.rlib.rarithmetic import ovfcheck
from rpython.rtyper.lltypesystem import llmemory
from rpython.jit.metainterp import history
@@ -390,8 +391,8 @@
val = op.getarg(0)
if val not in self.write_barrier_applied:
v = op.getarg(1)
- if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and
- bool(v.value)): # store a non-NULL
+ if (isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and
+ rgc.needs_write_barrier(v.value))):
self.gen_write_barrier(val)
#op = op.copy_and_change(rop.SETFIELD_RAW)
self.newops.append(op)
@@ -400,8 +401,8 @@
val = op.getarg(0)
if val not in self.write_barrier_applied:
v = op.getarg(2)
- if isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and
- bool(v.value)): # store a non-NULL
+ if (isinstance(v, BoxPtr) or (isinstance(v, ConstPtr) and
+ rgc.needs_write_barrier(v.value))):
self.gen_write_barrier_array(val, op.getarg(1))
#op = op.copy_and_change(rop.SET{ARRAYITEM,INTERIORFIELD}_RAW)
self.newops.append(op)
diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -38,6 +38,7 @@
FILEP = rffi.COpaquePtr("FILE")
OFF_T = config['off_t']
+
_IONBF = config['_IONBF']
_IOLBF = config['_IOLBF']
_IOFBF = config['_IOFBF']
@@ -47,6 +48,11 @@
BASE_BUF_SIZE = 4096
BASE_LINE_SIZE = 100
+NEWLINE_UNKNOWN = 0
+NEWLINE_CR = 1
+NEWLINE_LF = 2
+NEWLINE_CRLF = 4
+
def llexternal(*args, **kwargs):
return rffi.llexternal(*args, compilation_info=eci, **kwargs)
@@ -72,6 +78,7 @@
_pclose2 = (c_pclose, c_pclose_in_del)
c_getc = llexternal('getc', [FILEP], rffi.INT, macro=True)
+c_ungetc = llexternal('ungetc', [rffi.INT, FILEP], rffi.INT)
c_fgets = llexternal('fgets', [rffi.CCHARP, rffi.INT, FILEP], rffi.CCHARP)
c_fread = llexternal('fread', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
rffi.SIZE_T)
@@ -89,6 +96,10 @@
c_ferror = llexternal('ferror', [FILEP], rffi.INT)
c_clearerr = llexternal('clearerr', [FILEP], lltype.Void)
+c_stdin = rffi.CExternVariable(FILEP, 'stdin', eci, c_type='FILE*')[0]
+c_stdout = rffi.CExternVariable(FILEP, 'stdout', eci, c_type='FILE*')[0]
+c_stderr = rffi.CExternVariable(FILEP, 'stderr', eci, c_type='FILE*')[0]
+
def _error(ll_file):
err = c_ferror(ll_file)
@@ -128,10 +139,10 @@
def create_file(filename, mode="r", buffering=-1):
- mode = _sanitize_mode(mode)
+ newmode = _sanitize_mode(mode)
ll_name = rffi.str2charp(filename)
try:
- ll_mode = rffi.str2charp(mode)
+ ll_mode = rffi.str2charp(newmode)
try:
ll_file = c_fopen(ll_name, ll_mode)
if not ll_file:
@@ -142,29 +153,27 @@
finally:
lltype.free(ll_name, flavor='raw')
_dircheck(ll_file)
- if buffering >= 0:
- buf = lltype.nullptr(rffi.CCHARP.TO)
- if buffering == 0:
- c_setvbuf(ll_file, buf, _IONBF, 0)
- elif buffering == 1:
- c_setvbuf(ll_file, buf, _IOLBF, BUFSIZ)
- else:
- c_setvbuf(ll_file, buf, _IOFBF, buffering)
- return RFile(ll_file)
+ f = RFile(ll_file, mode)
+ f._setbufsize(buffering)
+ return f
-def create_fdopen_rfile(fd, mode="r"):
- mode = _sanitize_mode(mode)
- ll_mode = rffi.str2charp(mode)
+def create_fdopen_rfile(fd, mode="r", buffering=-1):
+ newmode = _sanitize_mode(mode)
+ fd = rffi.cast(rffi.INT, fd)
+ rposix.validate_fd(fd)
+ ll_mode = rffi.str2charp(newmode)
try:
- ll_file = c_fdopen(rffi.cast(rffi.INT, fd), ll_mode)
+ ll_file = c_fdopen(fd, ll_mode)
if not ll_file:
errno = rposix.get_errno()
raise OSError(errno, os.strerror(errno))
finally:
lltype.free(ll_mode, flavor='raw')
_dircheck(ll_file)
- return RFile(ll_file)
+ f = RFile(ll_file, mode)
+ f._setbufsize(buffering)
+ return f
def create_temp_rfile():
@@ -188,14 +197,46 @@
lltype.free(ll_type, flavor='raw')
finally:
lltype.free(ll_command, flavor='raw')
- return RFile(ll_file, _pclose2)
+ return RFile(ll_file, close2=_pclose2)
+
+
+def create_stdio():
+ close2 = (None, None)
+ stdin = RFile(c_stdin(), close2=close2)
+ stdout = RFile(c_stdout(), close2=close2)
+ stderr = RFile(c_stderr(), close2=close2)
+ return stdin, stdout, stderr
class RFile(object):
- def __init__(self, ll_file, close2=_fclose2):
+ _setbuf = lltype.nullptr(rffi.CCHARP.TO)
+ _univ_newline = False
+ _newlinetypes = NEWLINE_UNKNOWN
+ _skipnextlf = False
+
+ def __init__(self, ll_file, mode=None, close2=_fclose2):
self._ll_file = ll_file
+ if mode is not None:
+ self._univ_newline = 'U' in mode
self._close2 = close2
+ def _setbufsize(self, bufsize):
+ if bufsize >= 0:
+ if bufsize == 0:
+ mode = _IONBF
+ elif bufsize == 1:
+ mode = _IOLBF
+ bufsize = BUFSIZ
+ else:
+ mode = _IOFBF
+ if self._setbuf:
+ lltype.free(self._setbuf, flavor='raw')
+ if mode == _IONBF:
+ self._setbuf = lltype.nullptr(rffi.CCHARP.TO)
+ else:
+ self._setbuf = lltype.malloc(rffi.CCHARP.TO, bufsize, flavor='raw')
+ c_setvbuf(self._ll_file, self._setbuf, mode, bufsize)
+
def __del__(self):
"""Closes the described file when the object's last reference
goes away. Unlike an explicit call to close(), this is meant
@@ -204,7 +245,13 @@
ll_file = self._ll_file
if ll_file:
do_close = self._close2[1]
- do_close(ll_file) # return value ignored
+ if do_close:
+ do_close(ll_file) # return value ignored
+ if self._setbuf:
+ lltype.free(self._setbuf, flavor='raw')
+
+ def _cleanup_(self):
+ self._ll_file = lltype.nullptr(FILEP.TO)
def close(self):
"""Closes the described file.
@@ -220,16 +267,68 @@
# double close is allowed
self._ll_file = lltype.nullptr(FILEP.TO)
do_close = self._close2[0]
- res = do_close(ll_file)
- if res == -1:
- errno = rposix.get_errno()
- raise IOError(errno, os.strerror(errno))
+ try:
+ if do_close:
+ res = do_close(ll_file)
+ if res == -1:
+ errno = rposix.get_errno()
+ raise IOError(errno, os.strerror(errno))
+ finally:
+ if self._setbuf:
+ lltype.free(self._setbuf, flavor='raw')
+ self._setbuf = lltype.nullptr(rffi.CCHARP.TO)
return res
def _check_closed(self):
if not self._ll_file:
raise ValueError("I/O operation on closed file")
+ def _fread(self, buf, n, stream):
+ if not self._univ_newline:
+ return c_fread(buf, 1, n, stream)
+
+ i = 0
+ dst = buf
+ newlinetypes = self._newlinetypes
+ skipnextlf = self._skipnextlf
+ while n:
+ nread = c_fread(dst, 1, n, stream)
+ if nread == 0:
+ break
+
+ src = dst
+ n -= nread
+ shortread = n != 0
+ while nread:
+ nread -= 1
+ c = src[0]
+ src = rffi.ptradd(src, 1)
+ if c == '\r':
+ dst[0] = '\n'
+ dst = rffi.ptradd(dst, 1)
+ i += 1
+ skipnextlf = True
+ elif skipnextlf and c == '\n':
+ skipnextlf = False
+ newlinetypes |= NEWLINE_CRLF
+ n += 1
+ else:
+ if c == '\n':
+ newlinetypes |= NEWLINE_LF
+ elif skipnextlf:
+ newlinetypes |= NEWLINE_CR
+ dst[0] = c
+ dst = rffi.ptradd(dst, 1)
+ i += 1
+ skipnextlf = False
+ if shortread:
+ if skipnextlf and c_feof(stream):
+ newlinetypes |= NEWLINE_CR
+ break
+ self._newlinetypes = newlinetypes
+ self._skipnextlf = skipnextlf
+ return i
+
def read(self, size=-1):
# XXX CPython uses a more delicate logic here
self._check_closed()
@@ -242,7 +341,7 @@
try:
s = StringBuilder()
while True:
- returned_size = c_fread(buf, 1, BASE_BUF_SIZE, ll_file)
+ returned_size = self._fread(buf, BASE_BUF_SIZE, ll_file)
returned_size = intmask(returned_size) # is between 0 and BASE_BUF_SIZE
if returned_size == 0:
if c_feof(ll_file):
@@ -254,12 +353,13 @@
lltype.free(buf, flavor='raw')
else: # size > 0
with rffi.scoped_alloc_buffer(size) as buf:
- returned_size = c_fread(buf.raw, 1, size, ll_file)
+ returned_size = self._fread(buf.raw, size, ll_file)
returned_size = intmask(returned_size) # is between 0 and size
if returned_size == 0:
if not c_feof(ll_file):
raise _error(ll_file)
s = buf.str(returned_size)
+ assert s is not None
return s
def _readline1(self, raw_buf):
@@ -300,7 +400,7 @@
self._check_closed()
if size == 0:
return ""
- elif size < 0:
+ elif size < 0 and not self._univ_newline:
with rffi.scoped_alloc_buffer(BASE_LINE_SIZE) as buf:
c = self._readline1(buf.raw)
if c >= 0:
@@ -315,19 +415,50 @@
break
s.append_charpsize(buf.raw, c)
return s.build()
- else: # size > 0
+ else: # size > 0 or self._univ_newline
ll_file = self._ll_file
+ c = 0
s = StringBuilder()
- while s.getlength() < size:
- c = c_getc(ll_file)
+ if self._univ_newline:
+ newlinetypes = self._newlinetypes
+ skipnextlf = self._skipnextlf
+ while size < 0 or s.getlength() < size:
+ c = c_getc(ll_file)
+ if c == EOF:
+ break
+ if skipnextlf:
+ skipnextlf = False
+ if c == ord('\n'):
+ newlinetypes |= NEWLINE_CRLF
+ c = c_getc(ll_file)
+ if c == EOF:
+ break
+ else:
+ newlinetypes |= NEWLINE_CR
+ if c == ord('\r'):
+ skipnextlf = True
+ c = ord('\n')
+ elif c == ord('\n'):
+ newlinetypes |= NEWLINE_LF
+ s.append(chr(c))
+ if c == ord('\n'):
+ break
if c == EOF:
- if c_ferror(ll_file):
- raise _error(ll_file)
- break
- c = chr(c)
- s.append(c)
- if c == '\n':
- break
+ if skipnextlf:
+ newlinetypes |= NEWLINE_CR
+ self._newlinetypes = newlinetypes
+ self._skipnextlf = skipnextlf
+ else:
+ while s.getlength() < size:
+ c = c_getc(ll_file)
+ if c == EOF:
+ break
+ s.append(chr(c))
+ if c == ord('\n'):
+ break
+ if c == EOF:
+ if c_ferror(ll_file):
+ raise _error(ll_file)
return s.build()
@enforceargs(None, str)
@@ -341,6 +472,7 @@
bytes = c_fwrite(ll_value, 1, length, self._ll_file)
if bytes != length:
errno = rposix.get_errno()
+ c_clearerr(self._ll_file)
raise IOError(errno, os.strerror(errno))
finally:
rffi.free_nonmovingbuffer(value, ll_value)
@@ -368,6 +500,7 @@
if res == -1:
errno = rposix.get_errno()
raise IOError(errno, os.strerror(errno))
+ self._skipnextlf = False
def tell(self):
self._check_closed()
@@ -375,8 +508,20 @@
if res == -1:
errno = rposix.get_errno()
raise IOError(errno, os.strerror(errno))
+ if self._skipnextlf:
+ c = c_getc(self._ll_file)
+ if c == ord('\n'):
+ self._newlinetypes |= NEWLINE_CRLF
+ res += 1
+ self._skipnextlf = False
+ elif c != EOF:
+ c_ungetc(c, self._ll_file)
return res
def fileno(self):
self._check_closed()
return intmask(c_fileno(self._ll_file))
+
+ def isatty(self):
+ self._check_closed()
+ return os.isatty(c_fileno(self._ll_file))
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -86,6 +86,14 @@
collect(i)
i += 1
+def needs_write_barrier(obj):
+ """ We need to emit write barrier if the right hand of assignment
+ is in nursery, used by the JIT for handling set*_gc(Const)
+ """
+ if not obj:
+ return False
+ return can_move(obj)
+
def _heap_stats():
raise NotImplementedError # can't be run directly
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -119,7 +119,7 @@
return 1
def validate_fd(fd):
- return 1
+ pass
def closerange(fd_low, fd_high):
# this behaves like os.closerange() from Python 2.6.
diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py
--- a/rpython/rlib/test/test_rfile.py
+++ b/rpython/rlib/test/test_rfile.py
@@ -104,15 +104,53 @@
f()
self.interpret(f, [])
+ @py.test.mark.skipif("sys.platform == 'win32'")
+ # http://msdn.microsoft.com/en-us/library/86cebhfs.aspx
+ def test_fdopen_buffering_line(self):
+ fname = str(self.tmpdir.join('file_1a'))
+
+ def f():
+ g = open(fname, 'w')
+ f = os.fdopen(os.dup(g.fileno()), 'w', 1)
+ g.close()
+ f.write('dupa\ndupb')
+ f2 = open(fname, 'r')
+ assert f2.read() == 'dupa\n'
+ f.close()
+ assert f2.read() == 'dupb'
+ f2.close()
+
+ f()
+ self.interpret(f, [])
+
def test_open_buffering_full(self):
fname = str(self.tmpdir.join('file_1b'))
def f():
f = open(fname, 'w', 128)
- f.write('dupa')
+ f.write('dupa\ndupb')
f2 = open(fname, 'r')
assert f2.read() == ''
- f.write('z' * 5000)
+ f.write('z' * 120)
+ assert f2.read() != ''
+ f.close()
+ assert f2.read() != ''
+ f2.close()
+
+ f()
+ self.interpret(f, [])
+
+ def test_fdopen_buffering_full(self):
+ fname = str(self.tmpdir.join('file_1b'))
+
+ def f():
+ g = open(fname, 'w')
+ f = os.fdopen(os.dup(g.fileno()), 'w', 128)
+ g.close()
+ f.write('dupa\ndupb')
+ f2 = open(fname, 'r')
+ assert f2.read() == ''
+ f.write('z' * 120)
assert f2.read() != ''
f.close()
assert f2.read() != ''
@@ -132,10 +170,22 @@
pass
else:
assert False
+ try:
+ f.readline()
+ except IOError as e:
+ pass
+ else:
+ assert False
f.write("dupa\x00dupb")
f.close()
for mode in ['r', 'U']:
f2 = open(fname, mode)
+ try:
+ f2.write('z')
+ except IOError as e:
+ pass
+ else:
+ assert False
dupa = f2.read(0)
assert dupa == ""
dupa = f2.read()
@@ -177,6 +227,39 @@
f()
self.interpret(f, [])
+ def test_read_universal(self):
+ fname = str(self.tmpdir.join('read_univ'))
+ with open(fname, 'wb') as f:
+ f.write("dupa\ndupb\r\ndupc\rdupd")
+
+ def f():
+ f = open(fname, 'U')
+ assert f.read() == "dupa\ndupb\ndupc\ndupd"
+ assert f.read() == ""
+ f.seek(0)
+ assert f.read(10) == "dupa\ndupb\n"
+ assert f.read(42) == "dupc\ndupd"
+ assert f.read(1) == ""
+ f.seek(0)
+ assert f.readline() == "dupa\n"
+ assert f.tell() == 5
+ assert f.readline() == "dupb\n"
+ assert f.tell() == 11
+ assert f.readline() == "dupc\n"
+ assert f.tell() == 16
+ assert f.readline() == "dupd"
+ assert f.tell() == 20
+ assert f.readline() == ""
+ f.seek(0)
+ assert f.readline() == "dupa\n"
+ assert f.readline() == "dupb\n"
+ f.seek(4)
+ assert f.read(1) == "\n"
+ f.close()
+
+ f()
+ self.interpret(f, [])
+
def test_seek(self):
fname = str(self.tmpdir.join('file_4'))
@@ -215,6 +298,12 @@
new_fno = os.dup(f.fileno())
f2 = os.fdopen(new_fno, "w")
f.close()
+ try:
+ f2.read()
+ except IOError as e:
+ pass
+ else:
+ assert False
f2.write("xxx")
f2.close()
@@ -228,6 +317,7 @@
def f():
f = open(fname, "w")
+ assert not f.isatty()
try:
return f.fileno()
finally:
@@ -281,6 +371,14 @@
data = f.read()
assert data == "hello w"
f.close()
+ f = open(fname)
+ try:
+ f.truncate()
+ except IOError as e:
+ pass
+ else:
+ assert False
+ f.close()
f()
self.interpret(f, [])
@@ -291,6 +389,13 @@
cls.tmpdir = udir.join('test_rfile_direct')
cls.tmpdir.ensure(dir=True)
+ def test_stdio(self):
+ i, o, e = rfile.create_stdio()
+ o.write("test\n")
+ i.close()
+ o.close()
+ e.close()
+
def test_auto_close(self):
fname = str(self.tmpdir.join('file_auto_close'))
f = rfile.create_file(fname, 'w')
diff --git a/rpython/rtyper/module/ll_os_stat.py b/rpython/rtyper/module/ll_os_stat.py
--- a/rpython/rtyper/module/ll_os_stat.py
+++ b/rpython/rtyper/module/ll_os_stat.py
@@ -186,7 +186,10 @@
_name_struct_stat = '_stati64'
INCLUDES = ['sys/types.h', 'sys/stat.h', 'sys/statvfs.h']
else:
- _name_struct_stat = 'stat'
+ if sys.platform.startswith('linux'):
+ _name_struct_stat = 'stat64'
+ else:
+ _name_struct_stat = 'stat'
INCLUDES = ['sys/types.h', 'sys/stat.h', 'sys/statvfs.h', 'unistd.h']
compilation_info = ExternalCompilationInfo(
diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py
--- a/rpython/rtyper/rbuiltin.py
+++ b/rpython/rtyper/rbuiltin.py
@@ -259,9 +259,6 @@
return i1
return i2
-def rtype_Exception__init__(hop):
- hop.exception_cannot_occur()
-
def rtype_object__init__(hop):
hop.exception_cannot_occur()
@@ -269,17 +266,19 @@
hop.exception_cannot_occur()
v_self = hop.args_v[0]
r_self = hop.args_r[0]
- if hop.nb_args >= 2:
+ if hop.nb_args <= 2:
+ v_errno = hop.inputconst(lltype.Signed, 0)
+ if hop.nb_args == 2:
+ v_strerror = hop.inputarg(rstr.string_repr, arg=1)
+ r_self.setfield(v_self, 'strerror', v_strerror, hop.llops)
+ else:
v_errno = hop.inputarg(lltype.Signed, arg=1)
- else:
- v_errno = hop.inputconst(lltype.Signed, 0)
- r_self.setfield(v_self, 'errno', v_errno, hop.llops)
- if hop.nb_args >= 3:
v_strerror = hop.inputarg(rstr.string_repr, arg=2)
r_self.setfield(v_self, 'strerror', v_strerror, hop.llops)
if hop.nb_args >= 4:
v_filename = hop.inputarg(rstr.string_repr, arg=3)
r_self.setfield(v_self, 'filename', v_filename, hop.llops)
+ r_self.setfield(v_self, 'errno', v_errno, hop.llops)
def rtype_WindowsError__init__(hop):
hop.exception_cannot_occur()
@@ -339,6 +338,9 @@
original = getattr(__builtin__, name[14:])
BUILTIN_TYPER[original] = value
+BUILTIN_TYPER[getattr(object.__init__, 'im_func', object.__init__)] = (
+ rtype_object__init__)
+
BUILTIN_TYPER[getattr(EnvironmentError.__init__, 'im_func', EnvironmentError.__init__)] = (
rtype_EnvironmentError__init__)
@@ -351,8 +353,6 @@
getattr(WindowsError.__init__, 'im_func', WindowsError.__init__)] = (
rtype_WindowsError__init__)
-BUILTIN_TYPER[getattr(object.__init__, 'im_func', object.__init__)] = (
- rtype_object__init__)
# annotation of low-level types
def rtype_malloc(hop, i_flavor=None, i_zero=None, i_track_allocation=None,
diff --git a/rpython/rtyper/test/test_exception.py b/rpython/rtyper/test/test_exception.py
--- a/rpython/rtyper/test/test_exception.py
+++ b/rpython/rtyper/test/test_exception.py
@@ -36,7 +36,7 @@
class TestException(BaseRtypingTest):
def test_exception_with_arg(self):
def g(n):
- raise IOError(n)
+ raise IOError("test")
def h(n):
raise OSError(n, "?", None)
def i(n):
@@ -49,8 +49,8 @@
try:
g(n)
except IOError, e:
- assert e.errno == 42
- assert e.strerror is None
+ assert e.errno == 0
+ assert e.strerror == "test"
assert e.filename is None
else:
assert False
diff --git a/rpython/translator/platform/test/test_makefile.py b/rpython/translator/platform/test/test_makefile.py
--- a/rpython/translator/platform/test/test_makefile.py
+++ b/rpython/translator/platform/test/test_makefile.py
@@ -44,6 +44,7 @@
assert res.returncode == 0
def test_900_files(self):
+ tmpdir = udir.join('test_900_files').ensure(dir=1)
txt = '#include <stdio.h>\n'
for i in range(900):
txt += 'int func%03d();\n' % i
@@ -52,11 +53,11 @@
txt += ' j += func%03d();\n' % i
txt += ' printf("%d\\n", j);\n'
txt += ' return 0;};\n'
- cfile = udir.join('test_900_files.c')
+ cfile = tmpdir.join('test_900_files.c')
cfile.write(txt)
cfiles = [cfile]
for i in range(900):
- cfile2 = udir.join('implement%03d.c' %i)
+ cfile2 = tmpdir.join('implement%03d.c' %i)
cfile2.write('''
int func%03d()
{
@@ -64,10 +65,10 @@
}
''' % (i, i))
cfiles.append(cfile2)
- mk = self.platform.gen_makefile(cfiles, ExternalCompilationInfo(), path=udir)
+ mk = self.platform.gen_makefile(cfiles, ExternalCompilationInfo(), path=tmpdir)
mk.write()
self.platform.execute_makefile(mk)
- res = self.platform.execute(udir.join('test_900_files'))
+ res = self.platform.execute(tmpdir.join('test_900_files'))
self.check_res(res, '%d\n' %sum(range(900)))
def test_precompiled_headers(self):
diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -100,8 +100,8 @@
cc = 'cl.exe'
link = 'link.exe'
- cflags = ('/MD', '/O2')
- link_flags = ()
+ cflags = ('/MD', '/O2', '/Zi')
+ link_flags = ('/debug',)
standalone_only = ()
shared_only = ()
environ = None
@@ -143,7 +143,6 @@
# Install debug options only when interpreter is in debug mode
if sys.executable.lower().endswith('_d.exe'):
self.cflags = ['/MDd', '/Z7', '/Od']
- self.link_flags = ['/debug']
# Increase stack size, for the linker and the stack check code.
stack_size = 8 << 20 # 8 Mb
@@ -204,6 +203,9 @@
# the assembler still has the old behavior that all options
# must come first, and after the file name all options are ignored.
# So please be careful with the order of parameters! ;-)
+ pdb_dir = oname.dirname
+ if pdb_dir:
+ compile_args += ['/Fd%s\\' % (pdb_dir,)]
args = ['/nologo', '/c'] + compile_args + ['/Fo%s' % (oname,), str(cfile)]
self._execute_c_compiler(cc, args, oname)
return oname
diff --git a/rpython/translator/sandbox/rsandbox.py b/rpython/translator/sandbox/rsandbox.py
--- a/rpython/translator/sandbox/rsandbox.py
+++ b/rpython/translator/sandbox/rsandbox.py
@@ -60,26 +60,23 @@
def need_more_data(self):
buflen = self.buflen
- buf = lltype.malloc(rffi.CCHARP.TO, buflen, flavor='raw')
- buflen = rffi.cast(rffi.SIZE_T, buflen)
- count = ll_read_not_sandboxed(self.fd, buf, buflen)
- count = rffi.cast(lltype.Signed, count)
- if count <= 0:
- raise IOError
- self.buf += ''.join([buf[i] for i in range(count)])
- self.buflen *= 2
+ with lltype.scoped_alloc(rffi.CCHARP.TO, buflen) as buf:
+ buflen = rffi.cast(rffi.SIZE_T, buflen)
+ count = ll_read_not_sandboxed(self.fd, buf, buflen)
+ count = rffi.cast(lltype.Signed, count)
+ if count <= 0:
+ raise IOError
+ self.buf += ''.join([buf[i] for i in range(count)])
+ self.buflen *= 2
def sandboxed_io(buf):
STDIN = 0
STDOUT = 1
# send the buffer with the marshalled fnname and input arguments to STDOUT
- p = lltype.malloc(rffi.CCHARP.TO, len(buf), flavor='raw')
- try:
+ with lltype.scoped_alloc(rffi.CCHARP.TO, len(buf)) as p:
for i in range(len(buf)):
p[i] = buf[i]
writeall_not_sandboxed(STDOUT, p, len(buf))
- finally:
- lltype.free(p, flavor='raw')
# build a Loader that will get the answer from STDIN
loader = FdLoader(STDIN)
# check for errors
@@ -105,9 +102,8 @@
@signature(types.str(), returns=types.impossible())
def not_implemented_stub(msg):
STDERR = 2
- buf = rffi.str2charp(msg + '\n')
- writeall_not_sandboxed(STDERR, buf, len(msg) + 1)
- rffi.free_charp(buf)
+ with rffi.scoped_str2charp(msg + '\n') as buf:
+ writeall_not_sandboxed(STDERR, buf, len(msg) + 1)
raise RuntimeError(msg) # XXX in RPython, the msg is ignored at the moment
dump_string = rmarshal.get_marshaller(str)
More information about the pypy-commit
mailing list