[pypy-commit] pypy winmultiprocessing: Merge py3.6 branch
andrewjlawrence
pypy.commits at gmail.com
Sun May 26 15:37:49 EDT 2019
Author: andrewjlawrence
Branch: winmultiprocessing
Changeset: r96690:5833cf4a4240
Date: 2019-05-26 20:36 +0100
http://bitbucket.org/pypy/pypy/changeset/5833cf4a4240/
Log: Merge py3.6 branch
diff --git a/lib-python/2.7/test/capath/efa5f9c3.0 b/lib-python/2.7/test/capath/efa5f9c3.0
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/capath/efa5f9c3.0
@@ -0,0 +1,34 @@
+-----BEGIN CERTIFICATE-----
+MIIF9zCCA9+gAwIBAgIUH98b4Fw/DyugC9cV7VK7ZODzHsIwDQYJKoZIhvcNAQEL
+BQAwgYoxCzAJBgNVBAYTAlhZMRcwFQYDVQQIDA5DYXN0bGUgQW50aHJheDEYMBYG
+A1UEBwwPQXJndW1lbnQgQ2xpbmljMSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUg
+Rm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0aG9udGVzdC5uZXQw
+HhcNMTkwNTA4MDEwMjQzWhcNMjcwNzI0MDEwMjQzWjCBijELMAkGA1UEBhMCWFkx
+FzAVBgNVBAgMDkNhc3RsZSBBbnRocmF4MRgwFgYDVQQHDA9Bcmd1bWVudCBDbGlu
+aWMxIzAhBgNVBAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMSMwIQYDVQQD
+DBpzZWxmLXNpZ25lZC5weXRob250ZXN0Lm5ldDCCAiIwDQYJKoZIhvcNAQEBBQAD
+ggIPADCCAgoCggIBAMKdJlyCThkahwoBb7pl5q64Pe9Fn5jrIvzsveHTc97TpjV2
+RLfICnXKrltPk/ohkVl6K5SUZQZwMVzFubkyxE0nZPHYHlpiKWQxbsYVkYv01rix
+IFdLvaxxbGYke2jwQao31s4o61AdlsfK1SdpHQUynBBMssqI3SB4XPmcA7e+wEEx
+jxjVish4ixA1vuIZOx8yibu+CFCf/geEjoBMF3QPdzULzlrCSw8k/45iZCSoNbvK
+DoL4TVV07PHOxpheDh8ZQmepGvU6pVqhb9m4lgmV0OGWHgozd5Ur9CbTVDmxIEz3
+TSoRtNJK7qtyZdGNqwjksQxgZTjM/d/Lm/BJG99AiOmYOjsl9gbQMZgvQmMAtUsI
+aMJnQuZ6R+KEpW/TR5qSKLWZSG45z/op+tzI2m+cE6HwTRVAWbcuJxcAA55MZjqU
+OOOu3BBYMjS5nf2sQ9uoXsVBFH7i0mQqoW1SLzr9opI8KsWwFxQmO2vBxWYaN+lH
+OmwBZBwyODIsmI1YGXmTp09NxRYz3Qe5GCgFzYowpMrcxUC24iduIdMwwhRM7rKg
+7GtIWMSrFfuI1XCLRmSlhDbhNN6fVg2f8Bo9PdH9ihiIyxSrc+FOUasUYCCJvlSZ
+8hFUlLvcmrZlWuazohm0lsXuMK1JflmQr/DA/uXxP9xzFfRy+RU3jDyxJbRHAgMB
+AAGjUzBRMB0GA1UdDgQWBBSQJyxiPMRK01i+0BsV9zUwDiBaHzAfBgNVHSMEGDAW
+gBSQJyxiPMRK01i+0BsV9zUwDiBaHzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3
+DQEBCwUAA4ICAQCR+7a7N/m+WLkxPPIA/CB4MOr2Uf8ixTv435Nyv6rXOun0+lTP
+ExSZ0uYQ+L0WylItI3cQHULldDueD+s8TGzxf5woaLKf6tqyr0NYhKs+UeNEzDnN
+9PHQIhX0SZw3XyXGUgPNBfRCg2ZDdtMMdOU4XlQN/IN/9hbYTrueyY7eXq9hmtI9
+1srftAMqr9SR1JP7aHI6DVgrEsZVMTDnfT8WmLSGLlY1HmGfdEn1Ip5sbo9uSkiH
+AEPgPfjYIvR5LqTOMn4KsrlZyBbFIDh9Sl99M1kZzgH6zUGVLCDg1y6Cms69fx/e
+W1HoIeVkY4b4TY7Bk7JsqyNhIuqu7ARaxkdaZWhYaA2YyknwANdFfNpfH+elCLIk
+BUt5S3f4i7DaUePTvKukCZiCq4Oyln7RcOn5If73wCeLB/ZM9Ei1HforyLWP1CN8
+XLfpHaoeoPSWIveI0XHUl65LsPN2UbMbul/F23hwl+h8+BLmyAS680Yhn4zEN6Ku
+B7Po90HoFa1Du3bmx4jsN73UkT/dwMTi6K072FbipnC1904oGlWmLwvAHvrtxxmL
+Pl3pvEaZIu8wa/PNF6Y7J7VIewikIJq6Ta6FrWeFfzMWOj2qA1ZZi6fUaDSNYvuV
+J5quYKCc/O+I/yDDf8wyBbZ/gvUXzUHTMYGG+bFrn1p7XDbYYeEJ6R/xEg==
+-----END CERTIFICATE-----
diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py
--- a/lib-python/3/datetime.py
+++ b/lib-python/3/datetime.py
@@ -6,6 +6,7 @@
import time as _time
import math as _math
+import sys
# for cpyext, use these as base classes
from __pypy__._pypydatetime import dateinterop, deltainterop, timeinterop
@@ -1379,7 +1380,8 @@
def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0,
microsecond=0, tzinfo=None, *, fold=0):
- if isinstance(year, bytes) and len(year) == 10 and 1 <= year[2]&0x7F <= 12:
+ if (isinstance(year, (bytes, str)) and len(year) == 10 and
+ 1 <= ord(year[2:3])&0x7F <= 12):
# Pickle support
self = dateinterop.__new__(cls)
self.__setstate(year, month)
@@ -1456,6 +1458,14 @@
# 23 hours at 1969-09-30 13:00:00 in Kwajalein.
# Let's probe 24 hours in the past to detect a transition:
max_fold_seconds = 24 * 3600
+
+ # On Windows localtime_s throws an OSError for negative values,
+ # thus we can't perform fold detection for values of time less
+ # than the max time fold. See comments in _datetimemodule's
+ # version of this method for more details.
+ if t < max_fold_seconds and sys.platform.startswith("win"):
+ return result
+
y, m, d, hh, mm, ss = converter(t - max_fold_seconds)[:6]
probe1 = cls(y, m, d, hh, mm, ss, us, tz)
trans = result - probe1 - timedelta(0, max_fold_seconds)
diff --git a/lib-python/3/sysconfig.py b/lib-python/3/sysconfig.py
--- a/lib-python/3/sysconfig.py
+++ b/lib-python/3/sysconfig.py
@@ -563,6 +563,7 @@
_CONFIG_VARS['abiflags'] = ''
_CONFIG_VARS['implementation'] = _get_implementation()
_CONFIG_VARS['implementation_lower'] = _get_implementation().lower()
+ _CONFIG_VARS['LIBRARY'] = ''
if os.name == 'nt':
_init_non_posix(_CONFIG_VARS)
diff --git a/lib_pypy/_cffi_ssl/.gitignore b/lib_pypy/_cffi_ssl/.gitignore
deleted file mode 100644
--- a/lib_pypy/_cffi_ssl/.gitignore
+++ /dev/null
@@ -1,95 +0,0 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-env/
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-*.egg-info/
-.installed.cfg
-*.egg
-
-# PyInstaller
-# Usually these files are written by a python script from a template
-# before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*,cover
-.hypothesis/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# pyenv
-.python-version
-
-# celery beat schedule file
-celerybeat-schedule
-
-# dotenv
-.env
-
-# virtualenv
-.venv/
-venv/
-ENV/
-
-# Spyder project settings
-.spyderproject
-
-# Rope project settings
-.ropeproject
-
-# Vim
-
-*.swp
-*.swo
diff --git a/lib_pypy/_cffi_ssl/__init__.py b/lib_pypy/_cffi_ssl/__init__.py
new file mode 100644
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
@@ -71,6 +71,7 @@
static const long SSL_OP_MICROSOFT_SESS_ID_BUG;
static const long SSL_OP_NETSCAPE_CHALLENGE_BUG;
static const long SSL_OP_NETSCAPE_REUSE_CIPHER_CHANGE_BUG;
+static const long SSL_OP_NO_SSLv2;
static const long SSL_OP_SSLREF2_REUSE_CERT_TYPE_BUG;
static const long SSL_OP_MICROSOFT_BIG_SSLV3_BUFFER;
static const long SSL_OP_MSIE_SSLV2_RSA_PADDING;
diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -1,7 +1,6 @@
import sys
import time
import _thread
-import socket
import weakref
from _pypy_openssl import ffi
from _pypy_openssl import lib
@@ -70,6 +69,7 @@
globals()[name[4:]] = getattr(lib, name)
OP_ALL = lib.SSL_OP_ALL & ~lib.SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS
+OP_NO_SSLv2 = lib.SSL_OP_NO_SSLv2
SSL_CLIENT = 0
SSL_SERVER = 1
@@ -78,7 +78,8 @@
if lib.Cryptography_HAS_SSL2:
PROTOCOL_SSLv2 = 0
-PROTOCOL_SSLv3 = 1
+if lib.Cryptography_HAS_SSL3_METHOD:
+ PROTOCOL_SSLv3 = 1
PROTOCOL_SSLv23 = 2
PROTOCOL_TLS = PROTOCOL_SSLv23
PROTOCOL_TLSv1 = 3
@@ -310,6 +311,9 @@
return self.socket_type == SSL_SERVER
def do_handshake(self):
+ # delay to prevent circular imports
+ import socket
+
sock = self.get_socket_or_connection_gone()
ssl = self.ssl
timeout = _socket_timeout(sock)
@@ -381,6 +385,9 @@
return _decode_certificate(self.peer_cert)
def write(self, bytestring):
+ # delay to prevent circular imports
+ import socket
+
deadline = 0
b = _str_to_ffi_buffer(bytestring)
sock = self.get_socket_or_connection_gone()
@@ -439,6 +446,9 @@
raise pyssl_error(self, length)
def read(self, length, buffer_into=None):
+ # delay to prevent circular imports
+ import socket
+
ssl = self.ssl
if length < 0 and buffer_into is None:
@@ -579,6 +589,9 @@
return sock
def shutdown(self):
+ # delay to prevent circular imports
+ import socket
+
sock = self.get_socket_or_None()
nonblocking = False
ssl = self.ssl
@@ -804,7 +817,7 @@
method = lib.TLSv1_1_method()
elif lib.Cryptography_HAS_TLSv1_2 and protocol == PROTOCOL_TLSv1_2 :
method = lib.TLSv1_2_method()
- elif protocol == PROTOCOL_SSLv3 and lib.Cryptography_HAS_SSL3_METHOD:
+ elif lib.Cryptography_HAS_SSL3_METHOD and protocol == PROTOCOL_SSLv3:
method = lib.SSLv3_method()
elif lib.Cryptography_HAS_SSL2 and protocol == PROTOCOL_SSLv2:
method = lib.SSLv2_method()
@@ -835,7 +848,7 @@
options = lib.SSL_OP_ALL & ~lib.SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS
if not lib.Cryptography_HAS_SSL2 or protocol != PROTOCOL_SSLv2:
options |= lib.SSL_OP_NO_SSLv2
- if protocol != PROTOCOL_SSLv3:
+ if not lib.Cryptography_HAS_SSL3_METHOD or protocol != PROTOCOL_SSLv3:
options |= lib.SSL_OP_NO_SSLv3
# Minimal security flags for server and client side context.
# Client sockets ignore server-side parameters.
diff --git a/lib_pypy/_cffi_ssl/_stdssl/utility.py b/lib_pypy/_cffi_ssl/_stdssl/utility.py
--- a/lib_pypy/_cffi_ssl/_stdssl/utility.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/utility.py
@@ -19,7 +19,7 @@
elif isinstance(view, memoryview):
# NOTE pypy limitation StringBuffer does not allow
# to get a raw address to the string!
- view = bytes(view)
+ view = view.tobytes()
# dont call call ffi.from_buffer(bytes(view)), arguments
# like ints/bools should result in a TypeError
return ffi.from_buffer(view)
diff --git a/lib_pypy/_ssl/__init__.py b/lib_pypy/_ssl/__init__.py
--- a/lib_pypy/_ssl/__init__.py
+++ b/lib_pypy/_ssl/__init__.py
@@ -3,6 +3,8 @@
from _cffi_ssl import _stdssl
from _cffi_ssl._stdssl import *
+OP_SINGLE_DH_USE = lib.SSL_OP_SINGLE_DH_USE
+OP_SINGLE_ECDH_USE = lib.SSL_OP_SINGLE_ECDH_USE
try: from __pypy__ import builtinify
except ImportError: builtinify = lambda f: f
diff --git a/lib_pypy/_sysconfigdata.py b/lib_pypy/_sysconfigdata.py
--- a/lib_pypy/_sysconfigdata.py
+++ b/lib_pypy/_sysconfigdata.py
@@ -20,7 +20,7 @@
'AR': "ar",
'ARFLAGS': "rc",
'EXE': "",
- 'LIBDIR': os.path.join(sys.prefix, 'lib'),
+ 'LIBDIR': os.path.join(sys.prefix, 'bin'),
'VERSION': sys.version[:3]
}
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -113,42 +113,6 @@
module_suggests["cpyext"].append(("translation.shared", True))
-# NOTE: this dictionary is not used any more
-module_import_dependencies = {
- # no _rawffi if importing rpython.rlib.clibffi raises ImportError
- # or CompilationError or py.test.skip.Exception
- "_rawffi" : ["rpython.rlib.clibffi"],
-
- "zlib" : ["rpython.rlib.rzlib"],
- "bz2" : ["pypy.module.bz2.interp_bz2"],
- "pyexpat" : ["pypy.module.pyexpat.interp_pyexpat"],
- "_minimal_curses": ["pypy.module._minimal_curses.fficurses"],
- "_continuation": ["rpython.rlib.rstacklet"],
- "_vmprof" : ["pypy.module._vmprof.interp_vmprof"],
- "faulthandler" : ["pypy.module._vmprof.interp_vmprof"],
- "_lzma" : ["pypy.module._lzma.interp_lzma"],
- }
-
-def get_module_validator(modname):
- # NOTE: this function is not used any more
- if modname in module_import_dependencies:
- modlist = module_import_dependencies[modname]
- def validator(config):
- from rpython.rtyper.tool.rffi_platform import CompilationError
- try:
- for name in modlist:
- __import__(name)
- except (ImportError, CompilationError, py.test.skip.Exception) as e:
- errcls = e.__class__.__name__
- raise Exception(
- "The module %r is disabled\n" % (modname,) +
- "because importing %s raised %s\n" % (name, errcls) +
- str(e))
- return validator
- else:
- return None
-
-
pypy_optiondescription = OptionDescription("objspace", "Object Space Options", [
OptionDescription("usemodules", "Which Modules should be used", [
BoolOption(modname, "use module %s" % (modname, ),
@@ -157,7 +121,7 @@
requires=module_dependencies.get(modname, []),
suggests=module_suggests.get(modname, []),
negation=modname not in essential_modules,
- ) #validator=get_module_validator(modname))
+ )
for modname in all_modules]),
BoolOption("allworkingmodules", "use as many working modules as possible",
diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -8,6 +8,12 @@
a branch named like release-pypy3.5-v2.x or release-pypy3.5-v4.x, and each
release is tagged, for instance release-pypy3.5-v4.0.1.
+The release version number should be bumped. A micro release increment means
+there were no changes that justify rebuilding c-extension wheels, since
+the wheels are marked with only major.minor version numbers. It is ofen not
+clear what constitues a "major" release verses a "minor" release, the release
+manager can make that call.
+
After release, inevitably there are bug fixes. It is the responsibility of
the commiter who fixes a bug to make sure this fix is on the release branch,
so that we can then create a tagged bug-fix release, which will hopefully
@@ -78,6 +84,8 @@
* Maybe bump the SOABI number in module/imp/importing. This has many
implications, so make sure the PyPy community agrees to the change.
+ Wheels will use the major.minor release numbers in the name, so bump
+ them if there is an incompatible change to cpyext.
* Update and write documentation
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -25,3 +25,12 @@
Test and reduce the probability of a deadlock when acquiring a semaphore by
moving global state changes closer to the actual aquire.
+
+.. branch: shadowstack-issue2722
+
+Make the shadowstack size more dynamic
+
+.. branch: cffi-libs
+
+Move _ssl and _hashlib from rpython to a cffi-based module, like on python3.
+Reduces the number of problematic linked-in libraries (libssl, libcrypto)
diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -2,14 +2,9 @@
What's new in PyPy3 7.1+
========================
-.. this is the revision after release-pypy3.6-v7.1
-.. startrev: d642a3c217cb
+.. this is the revision after release-pypy3.6-v7.1.1
+.. startrev: db5a1e7fbbd0
-.. branch: zlib-make-py3-go-boom
+.. branch: fix-literal-prev_digit-underscore
-Complain if you try to copy a flushed zlib decompress on py3
-
-.. branch: winoverlapped
-
-Add support for async (overlapped) IO on Windows.
-
+Fix parsing for converting strings with underscore into ints
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -126,6 +126,9 @@
for c in expressions.constants:
yield (self.simple_test, "x="+c, "x", eval(c))
+ def test_const_underscore(self):
+ yield (self.simple_test, "x=0xffff_ffff_ff20_0000", "x", 0xffffffffff200000)
+
def test_neg_sys_maxint(self):
import sys
stmt = "x = %s" % (-sys.maxint-1)
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -384,6 +384,16 @@
PyAPI_FUNC(int) PyObject_CallFinalizerFromDealloc(PyObject *);
#endif
+/*
+ * On CPython with Py_REF_DEBUG these use _PyRefTotal, _Py_NegativeRefcount,
+ * _Py_GetRefTotal, ...
+ * So far we ignore Py_REF_DEBUG
+ */
+
+#define _Py_INC_REFTOTAL
+#define _Py_DEC_REFTOTAL
+#define _Py_REF_DEBUG_COMMA
+#define _Py_CHECK_REFCNT(OP) /* a semicolon */;
/* PyPy internal ----------------------------------- */
PyAPI_FUNC(int) PyPyType_Register(PyTypeObject *);
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -1462,7 +1462,7 @@
with open(fname, "w") as f:
f.write("this is a rename test")
str_name = str(self.pdir) + '/test_rename.txt'
- os.rename(self.path, str_name)
+ os.rename(fname, str_name)
with open(str_name) as f:
assert f.read() == 'this is a rename test'
os.rename(str_name, fname)
@@ -1471,6 +1471,11 @@
with open(unicode_name) as f:
assert f.read() == 'this is a rename test'
os.rename(unicode_name, fname)
+
+ os.rename(bytes(fname, 'utf-8'), bytes(str_name, 'utf-8'))
+ with open(str_name) as f:
+ assert f.read() == 'this is a rename test'
+ os.rename(str_name, fname)
with open(fname) as f:
assert f.read() == 'this is a rename test'
os.unlink(fname)
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -55,9 +55,15 @@
reserves 768KB of stack space, which should suffice (on Linux,
depending on the compiler settings) for ~1400 calls. Setting the
value to N reserves N/1000 times 768KB of stack space.
+
+Note that there are other factors that also limit the stack size.
+The operating system typically sets a maximum which can be changed
+manually (e.g. with "ulimit" on Linux) for the main thread. For other
+threads you can configure the limit by calling "threading.stack_size()".
"""
from rpython.rlib.rstack import _stack_set_length_fraction
from rpython.rlib.rstackovf import StackOverflow
+ from rpython.rlib.rgc import increase_root_stack_depth
if new_limit <= 0:
raise oefmt(space.w_ValueError, "recursion limit must be positive")
try:
@@ -69,6 +75,7 @@
raise oefmt(space.w_RecursionError,
"maximum recursion depth exceeded")
space.sys.recursionlimit = new_limit
+ increase_root_stack_depth(int(new_limit * 0.001 * 163840))
def getrecursionlimit(space):
"""Return the last value set by setrecursionlimit().
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -57,7 +57,9 @@
w_object._check_released(space)
return W_MemoryView.copy(w_object)
view = space.buffer_w(w_object, space.BUF_FULL_RO)
- return view.wrap(space)
+ mv = view.wrap(space)
+ mv.obj = w_object
+ return mv
def _make_descr__cmp(name):
def descr__cmp(self, space, w_other):
@@ -265,6 +267,10 @@
# I've never seen anyone filling this field
return space.newtuple([])
+ def w_get_obj(self, space):
+ self._check_released(space)
+ return self.obj
+
def descr_repr(self, space):
if self.view is None:
return self.getrepr(space, 'released memory')
@@ -529,6 +535,7 @@
shape = GetSetProperty(W_MemoryView.w_get_shape),
strides = GetSetProperty(W_MemoryView.w_get_strides),
suboffsets = GetSetProperty(W_MemoryView.w_get_suboffsets),
+ obj = GetSetProperty(W_MemoryView.w_get_obj),
_pypy_raw_address = interp2app(W_MemoryView.descr_pypy_raw_address),
)
W_MemoryView.typedef.acceptable_as_base_class = False
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -62,13 +62,15 @@
assert w.tobytes() == bytes(w) == b'geb'
def test_memoryview_attrs(self):
- v = memoryview(b"a"*100)
+ b = b"a"*100
+ v = memoryview(b)
assert v.format == "B"
assert v.itemsize == 1
assert v.shape == (100,)
assert v.ndim == 1
assert v.strides == (1,)
assert v.nbytes == 100
+ assert v.obj is b
def test_suboffsets(self):
v = memoryview(b"a"*100)
diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py
--- a/pypy/tool/build_cffi_imports.py
+++ b/pypy/tool/build_cffi_imports.py
@@ -14,7 +14,7 @@
"tk": "_tkinter/tklib_build.py",
"curses": "_curses_build.py" if sys.platform != "win32" else None,
"syslog": "_syslog_build.py" if sys.platform != "win32" else None,
- "_gdbm": "_gdbm_build.py" if sys.platform != "win32" else None,
+ "gdbm": "_gdbm_build.py" if sys.platform != "win32" else None,
"pwdgrp": "_pwdgrp_build.py" if sys.platform != "win32" else None,
"resource": "_resource_build.py" if sys.platform != "win32" else None,
"lzma": "_lzma_build.py",
diff --git a/pypy/tool/pytest/test/test_appsupport.py b/pypy/tool/pytest/test/test_appsupport.py
--- a/pypy/tool/pytest/test/test_appsupport.py
+++ b/pypy/tool/pytest/test/test_appsupport.py
@@ -37,25 +37,6 @@
class TestSpaceConfig:
@pytest.mark.xfail(reason="Can't check config with -A in pypy3")
- def test_applevel_skipped_on_cpython_and_spaceconfig(self, testdir):
- setpypyconftest(testdir)
- testdir.makepyfile("""
- class AppTestClass:
- spaceconfig = {"objspace.usemodules._random": True}
- def setup_class(cls):
- assert 0
- def test_applevel(self):
- pass
- """)
- result = testdir.runpytest("-A")
- assert result.ret == 0
- if hasattr(sys, 'pypy_translation_info') and \
- sys.pypy_translation_info.get('objspace.usemodules._random'):
- result.stdout.fnmatch_lines(["*1 error*"])
- else:
- # setup_class didn't get called, otherwise it would error
- result.stdout.fnmatch_lines(["*1 skipped*"])
-
def test_interp_spaceconfig(self, testdir):
setpypyconftest(testdir)
p = testdir.makepyfile("""
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -572,6 +572,8 @@
self.move_out_of_nursery_ptr = getfn(GCClass.move_out_of_nursery,
[s_gc, SomeAddress()],
SomeAddress())
+ if hasattr(self.root_walker, 'build_increase_root_stack_depth_ptr'):
+ self.root_walker.build_increase_root_stack_depth_ptr(getfn)
def create_custom_trace_funcs(self, gc, rtyper):
@@ -1652,6 +1654,12 @@
else:
hop.rename("same_as")
+ def gct_gc_increase_root_stack_depth(self, hop):
+ if not hasattr(self.root_walker, 'gc_increase_root_stack_depth_ptr'):
+ return
+ hop.genop("direct_call",
+ [self.root_walker.gc_increase_root_stack_depth_ptr,
+ hop.spaceop.args[0]])
class TransformerLayoutBuilder(gctypelayout.TypeLayoutBuilder):
diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py
--- a/rpython/memory/gctransform/shadowstack.py
+++ b/rpython/memory/gctransform/shadowstack.py
@@ -140,6 +140,7 @@
# this is a dict {tid: SHADOWSTACKREF}, where the tid for the
# current thread may be missing so far
gcdata.thread_stacks = None
+ shadow_stack_pool.has_threads = True
# Return the thread identifier, as an integer.
get_tid = rthread.get_ident
@@ -252,6 +253,15 @@
self.gc_modified_shadowstack_ptr = getfn(gc_modified_shadowstack,
[], annmodel.s_None)
+ def build_increase_root_stack_depth_ptr(self, getfn):
+ shadow_stack_pool = self.shadow_stack_pool
+ def gc_increase_root_stack_depth(new_size):
+ shadow_stack_pool.increase_root_stack_depth(new_size)
+
+ self.gc_increase_root_stack_depth_ptr = getfn(
+ gc_increase_root_stack_depth, [annmodel.SomeInteger()],
+ annmodel.s_None)
+
def postprocess_graph(self, gct, graph, any_inlining):
from rpython.memory.gctransform import shadowcolor
if any_inlining:
@@ -269,6 +279,7 @@
"""
_alloc_flavor_ = "raw"
root_stack_depth = 163840
+ has_threads = False
def __init__(self, gcdata):
self.unused_full_stack = llmemory.NULL
@@ -337,6 +348,44 @@
if self.unused_full_stack == llmemory.NULL:
raise MemoryError
+ def increase_root_stack_depth(self, new_depth):
+ if new_depth <= self.root_stack_depth:
+ return # can't easily decrease the size
+ if self.unused_full_stack:
+ llmemory.raw_free(self.unused_full_stack)
+ self.unused_full_stack = llmemory.NULL
+ used = self.gcdata.root_stack_top - self.gcdata.root_stack_base
+ addr = self._resize(self.gcdata.root_stack_base, used, new_depth)
+ self.gcdata.root_stack_base = addr
+ self.gcdata.root_stack_top = addr + used
+ # no gc operations above: we just switched shadowstacks
+ if self.has_threads:
+ self._resize_thread_shadowstacks(new_depth)
+ self.root_stack_depth = new_depth
+
+ def _resize_thread_shadowstacks(self, new_depth):
+ if self.gcdata.thread_stacks is not None:
+ for ssref in self.gcdata.thread_stacks.values():
+ if ssref.base:
+ used = ssref.top - ssref.base
+ addr = self._resize(ssref.base, used, new_depth)
+ ssref.base = addr
+ ssref.top = addr + used
+ _resize_thread_shadowstacks._dont_inline_ = True
+
+ def _resize(self, base, used, new_depth):
+ new_size = sizeofaddr * new_depth
+ ll_assert(used <= new_size, "shadowstack resize: overflow detected")
+ addr = llmemory.raw_malloc(new_size)
+ if addr == llmemory.NULL:
+ raise MemoryError
+ # note that we don't know the total memory size of 'base', but we
+ # know the size of the part that is used right now, and we only need
+ # to copy that
+ llmemory.raw_memmove(base, addr, used)
+ llmemory.raw_free(base)
+ return addr
+
def get_shadowstackref(root_walker, gctransformer):
if hasattr(gctransformer, '_SHADOWSTACKREF'):
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -690,6 +690,13 @@
hop.exception_cannot_occur()
return hop.genop('gc_move_out_of_nursery', hop.args_v, resulttype=hop.r_result)
+ at jit.dont_look_inside
+def increase_root_stack_depth(new_depth):
+ """Shadowstack: make sure the size of the shadowstack is at least
+ 'new_depth' pointers."""
+ from rpython.rtyper.lltypesystem.lloperation import llop
+ llop.gc_increase_root_stack_depth(lltype.Void, new_depth)
+
# ____________________________________________________________
diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -579,6 +579,11 @@
assert i >= 0
self.i = i
c = self.s[i]
+ if self.allow_underscores and c == '_':
+ i = self.i - 1
+ assert i >= 0
+ self.i = i
+ c = self.s[i]
digit = ord(c)
if '0' <= c <= '9':
digit -= ord('0')
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -1226,6 +1226,9 @@
def op_gc_move_out_of_nursery(self, obj):
raise NotImplementedError("gc_move_out_of_nursery")
+ def op_gc_increase_root_stack_depth(self, new_depth):
+ raise NotImplementedError("gc_increase_root_stack_depth")
+
def op_revdb_stop_point(self, *args):
pass
def op_revdb_send_answer(self, *args):
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -530,6 +530,7 @@
'gc_rawrefcount_next_dead': LLOp(),
'gc_move_out_of_nursery': LLOp(),
+ 'gc_increase_root_stack_depth': LLOp(canrun=True),
'gc_push_roots' : LLOp(), # temporary: list of roots to save
'gc_pop_roots' : LLOp(), # temporary: list of roots to restore
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -776,6 +776,9 @@
def op_gc_move_out_of_nursery(obj):
return obj
+def op_gc_increase_root_stack_depth(new_depth):
+ pass
+
def op_revdb_do_next_call():
pass
diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py
--- a/rpython/translator/c/gc.py
+++ b/rpython/translator/c/gc.py
@@ -450,9 +450,8 @@
# XXX hard-code the field name here
gcpol_ss = '%s->gcd_inst_root_stack_top' % funcgen.expr(c_gcdata)
#
- yield ('typedef struct { void %s; } pypy_ss_t;'
+ yield ('typedef struct { char %s; } pypy_ss_t;'
% ', '.join(['*s%d' % i for i in range(numcolors)]))
- yield 'pypy_ss_t *ss;'
funcgen.gcpol_ss = gcpol_ss
def OP_GC_PUSH_ROOTS(self, funcgen, op):
@@ -462,26 +461,29 @@
raise Exception("gc_pop_roots should be removed by postprocess_graph")
def OP_GC_ENTER_ROOTS_FRAME(self, funcgen, op):
- return 'ss = (pypy_ss_t *)%s; %s = (void *)(ss+1);' % (
- funcgen.gcpol_ss, funcgen.gcpol_ss)
+ # avoid arithmatic on void*
+ return '({0}) = (char*)({0}) + sizeof(pypy_ss_t);'.format(funcgen.gcpol_ss,)
def OP_GC_LEAVE_ROOTS_FRAME(self, funcgen, op):
- return '%s = (void *)ss;' % funcgen.gcpol_ss
+ # avoid arithmatic on void*
+ return '({0}) = (char*)({0}) - sizeof(pypy_ss_t);'.format(funcgen.gcpol_ss,)
def OP_GC_SAVE_ROOT(self, funcgen, op):
num = op.args[0].value
exprvalue = funcgen.expr(op.args[1])
- return 'ss->s%d = (void *)%s;\t/* gc_save_root */' % (num, exprvalue)
+ return '((pypy_ss_t *)%s)[-1].s%d = (char *)%s;' % (
+ funcgen.gcpol_ss, num, exprvalue)
def OP_GC_RESTORE_ROOT(self, funcgen, op):
num = op.args[0].value
exprvalue = funcgen.expr(op.args[1])
typename = funcgen.db.gettype(op.args[1].concretetype)
- result = '%s = (%s)ss->s%d;' % (exprvalue, cdecl(typename, ''), num)
+ result = '%s = (%s)((pypy_ss_t *)%s)[-1].s%d;' % (
+ exprvalue, cdecl(typename, ''), funcgen.gcpol_ss, num)
if isinstance(op.args[1], Constant):
- return '/* %s\t* gc_restore_root */' % result
+ return '/* %s */' % result
else:
- return '%s\t/* gc_restore_root */' % result
+ return result
class AsmGcRootFrameworkGcPolicy(BasicFrameworkGcPolicy):
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1912,6 +1912,45 @@
def test_total_gc_time(self):
res = self.run("total_gc_time")
assert res > 0 # should take a few microseconds
+
+ def define_increase_root_stack_depth(cls):
+ class X:
+ pass
+ def g(n):
+ if n <= 0:
+ return None
+ x = X()
+ x.n = n
+ x.next = g(n - 1)
+ return x
+ def f(depth):
+ from rpython.rlib.rstack import _stack_set_length_fraction
+ _stack_set_length_fraction(50.0)
+ # ^^^ the default is enough for at least 10'000 (but less than
+ # 100'000) recursions of the simple function g(). We multiply
+ # it by 50.0 to make sure that 200'000 works. The default
+ # shadowstack depth is 163'840 entries, so 200'000 overflows
+ # that default shadowstack depth, and gives a segfault unless
+ # the following line works too.
+ from rpython.rlib.rgc import increase_root_stack_depth
+ increase_root_stack_depth(depth + 100)
+ #
+ g(depth)
+ return 42
+ return f
+
+ def test_increase_root_stack_depth(self):
+ if not sys.platform.startswith('linux'):
+ py.test.skip("linux only")
+ #
+ def myrunner(args):
+ args1 = ['/bin/bash', '-c', 'ulimit -s unlimited && %s' %
+ (' '.join(args),)]
+ return subprocess.check_output(args1)
+ res = self.run("increase_root_stack_depth", 200000, runner=myrunner)
+ assert res == 42
+
+
# ____________________________________________________________________
class TaggedPointersTest(object):
diff --git a/rpython/translator/platform/arch/s390x.py b/rpython/translator/platform/arch/s390x.py
old mode 100644
new mode 100755
--- a/rpython/translator/platform/arch/s390x.py
+++ b/rpython/translator/platform/arch/s390x.py
@@ -88,6 +88,8 @@
return "zEC12"
if machine == 0x2964:
return "z13"
+ if machine == 0x3907: # gcc supports z14 as of 2019/05/08
+ return "z14"
# well all others are unsupported!
return "unknown"
More information about the pypy-commit
mailing list