[pypy-commit] pypy py3tests: hg merge py3.5
rlamy
pypy.commits at gmail.com
Sun Jul 29 06:59:55 EDT 2018
Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3tests
Changeset: r94926:b6b0feb7e7f0
Date: 2018-07-29 11:58 +0100
http://bitbucket.org/pypy/pypy/changeset/b6b0feb7e7f0/
Log: hg merge py3.5
diff too long, truncating to 2000 out of 25451 lines
diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -33,7 +33,12 @@
050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
+4909c06daf41ce88f87dc01c57959cadad4df4a8 RevDB-pypy2.7-v5.4.1
+4909c06daf41ce88f87dc01c57959cadad4df4a8 RevDB-pypy2.7-v5.4.1
+d7724c0a5700b895a47de44074cdf5fd659a988f RevDB-pypy2.7-v5.4.1
aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0
+e90317857d27917bf840caf675832292ee070510 RevDB-pypy2.7-v5.6.1
+a24d6c7000c8099c73d3660857f7e3cee5ac045c RevDB-pypy2.7-v5.6.2
fa3249d55d15b9829e1be69cdf45b5a44cec902d release-pypy2.7-v5.7.0
b16a4363e930f6401bceb499b9520955504c6cb0 release-pypy3.5-v5.7.0
1aa2d8e03cdfab54b7121e93fda7e98ea88a30bf release-pypy2.7-v5.7.1
@@ -51,3 +56,5 @@
0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0
09f9160b643e3f02ccb8c843b2fbb4e5cbf54082 release-pypy3.5-v5.10.0
3f6eaa010fce78cc7973bdc1dfdb95970f08fed2 release-pypy3.5-v5.10.1
+ab0b9caf307db6592905a80b8faffd69b39005b8 release-pypy2.7-v6.0.0
+fdd60ed87e941677e8ea11acf9f1819466521bf2 release-pypy3.5-v6.0.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -247,6 +247,7 @@
Lukas Vacek
Omer Katz
Jacek Generowicz
+ Tomasz Dziopa
Sylvain Thenault
Jakub Stasiak
Andrew Dalke
@@ -307,6 +308,7 @@
Yury V. Zaytsev
florinpapa
Anders Sigfridsson
+ Matt Jackson
Nikolay Zinov
rafalgalczynski at gmail.com
Joshua Gilbert
diff --git a/dotviewer/font/NOTICE b/dotviewer/font/COPYING.txt
rename from dotviewer/font/NOTICE
rename to dotviewer/font/COPYING.txt
diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py
--- a/lib-python/3/datetime.py
+++ b/lib-python/3/datetime.py
@@ -7,6 +7,9 @@
import time as _time
import math as _math
+# for cpyext, use these as base classes
+from __pypy__._pypydatetime import dateinterop, deltainterop, timeinterop
+
def _cmp(x, y):
return 0 if x == y else 1 if x > y else -1
@@ -316,7 +319,7 @@
return q
-class timedelta:
+class timedelta(deltainterop):
"""Represent the difference between two datetime objects.
Supported operators:
@@ -429,7 +432,7 @@
if abs(d) > 999999999:
raise OverflowError("timedelta # of days is too large: %d" % d)
- self = object.__new__(cls)
+ self = deltainterop.__new__(cls)
self._days = d
self._seconds = s
self._microseconds = us
@@ -638,7 +641,7 @@
microseconds=999999)
timedelta.resolution = timedelta(microseconds=1)
-class date:
+class date(dateinterop):
"""Concrete date type.
Constructors:
@@ -678,12 +681,12 @@
if month is None and isinstance(year, bytes) and len(year) == 4 and \
1 <= year[2] <= 12:
# Pickle support
- self = object.__new__(cls)
+ self = dateinterop.__new__(cls)
self.__setstate(year)
self._hashcode = -1
return self
year, month, day = _check_date_fields(year, month, day)
- self = object.__new__(cls)
+ self = dateinterop.__new__(cls)
self._year = year
self._month = month
self._day = day
@@ -1008,7 +1011,7 @@
_tzinfo_class = tzinfo
-class time:
+class time(timeinterop):
"""Time with time zone.
Constructors:
@@ -1044,14 +1047,14 @@
"""
if isinstance(hour, bytes) and len(hour) == 6 and hour[0] < 24:
# Pickle support
- self = object.__new__(cls)
+ self = timeinterop.__new__(cls)
self.__setstate(hour, minute or None)
self._hashcode = -1
return self
hour, minute, second, microsecond = _check_time_fields(
hour, minute, second, microsecond)
_check_tzinfo_arg(tzinfo)
- self = object.__new__(cls)
+ self = timeinterop.__new__(cls)
self._hour = hour
self._minute = minute
self._second = second
@@ -1329,7 +1332,7 @@
microsecond=0, tzinfo=None):
if isinstance(year, bytes) and len(year) == 10 and 1 <= year[2] <= 12:
# Pickle support
- self = object.__new__(cls)
+ self = dateinterop.__new__(cls)
self.__setstate(year, month)
self._hashcode = -1
return self
@@ -1337,7 +1340,7 @@
hour, minute, second, microsecond = _check_time_fields(
hour, minute, second, microsecond)
_check_tzinfo_arg(tzinfo)
- self = object.__new__(cls)
+ self = dateinterop.__new__(cls)
self._year = year
self._month = month
self._day = day
diff --git a/lib-python/3/distutils/sysconfig_pypy.py b/lib-python/3/distutils/sysconfig_pypy.py
--- a/lib-python/3/distutils/sysconfig_pypy.py
+++ b/lib-python/3/distutils/sysconfig_pypy.py
@@ -63,39 +63,9 @@
def _init_posix():
"""Initialize the module as appropriate for POSIX systems."""
- so_ext = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
-
- g = {}
- g['CC'] = "cc -pthread"
- g['CXX'] = "c++ -pthread"
- g['OPT'] = "-DNDEBUG -O2"
- g['CFLAGS'] = "-DNDEBUG -O2"
- g['CCSHARED'] = "-fPIC"
- g['LDSHARED'] = "cc -pthread -shared"
- g['EXT_SUFFIX'] = so_ext
- g['SHLIB_SUFFIX'] = ".so"
- g['SO'] = so_ext # deprecated in Python 3, for backward compatibility
- g['AR'] = "ar"
- g['ARFLAGS'] = "rc"
- g['EXE'] = ""
- g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
- g['VERSION'] = get_python_version()
-
- if sys.platform[:6] == "darwin":
- import platform
- if platform.machine() == 'i386':
- if platform.architecture()[0] == '32bit':
- arch = 'i386'
- else:
- arch = 'x86_64'
- else:
- # just a guess
- arch = platform.machine()
- g['LDSHARED'] += ' -undefined dynamic_lookup'
- g['CC'] += ' -arch %s' % (arch,)
-
+ from _sysconfigdata import build_time_vars
global _config_vars
- _config_vars = g
+ _config_vars = build_time_vars
def _init_nt():
@@ -221,4 +191,3 @@
from .sysconfig_cpython import (
parse_makefile, _variable_rx, expand_makefile_vars)
-
diff --git a/lib-python/3/fractions.py b/lib-python/3/fractions.py
--- a/lib-python/3/fractions.py
+++ b/lib-python/3/fractions.py
@@ -566,7 +566,7 @@
else:
return Fraction(round(self / shift) * shift)
- def __hash__(self):
+ def __hash__(self, CACHE=[-1] * 1001):
"""hash(self)"""
# XXX since this method is expensive, consider caching the result
@@ -580,12 +580,23 @@
# dinv is the inverse of self._denominator modulo the prime
# _PyHASH_MODULUS, or 0 if self._denominator is divisible by
# _PyHASH_MODULUS.
- dinv = pow(self._denominator, _PyHASH_MODULUS - 2, _PyHASH_MODULUS)
+
+ # PyPy3: added caching of pow() results for denominators up to 1000
+ denom = self._denominator
+ assert denom >= 0
+ if denom < len(CACHE):
+ dinv = CACHE[denom]
+ if dinv == -1:
+ dinv = pow(denom, _PyHASH_MODULUS - 2, _PyHASH_MODULUS)
+ CACHE[denom] = dinv
+ else:
+ dinv = pow(denom, _PyHASH_MODULUS - 2, _PyHASH_MODULUS)
+
if not dinv:
hash_ = _PyHASH_INF
else:
hash_ = abs(self._numerator) * dinv % _PyHASH_MODULUS
- result = hash_ if self >= 0 else -hash_
+ result = hash_ if self._numerator >= 0 else -hash_
return -2 if result == -1 else result
def __eq__(a, b):
diff --git a/lib-python/3/hashlib.py b/lib-python/3/hashlib.py
--- a/lib-python/3/hashlib.py
+++ b/lib-python/3/hashlib.py
@@ -134,9 +134,14 @@
__get_hash = __get_openssl_constructor
algorithms_available = algorithms_available.union(
_hashlib.openssl_md_meth_names)
-except ImportError:
+except ImportError as e:
new = __py_new
__get_hash = __get_builtin_constructor
+ # added by PyPy
+ import warnings
+ warnings.warn("The _hashlib module is not available, falling back "
+ "to a much slower implementation (%s)" % str(e),
+ RuntimeWarning)
try:
# OpenSSL's PKCS5_PBKDF2_HMAC requires OpenSSL 1.0+ with HMAC and SHA
diff --git a/lib-python/3/opcode.py b/lib-python/3/opcode.py
--- a/lib-python/3/opcode.py
+++ b/lib-python/3/opcode.py
@@ -223,5 +223,6 @@
def_op('CALL_METHOD', 202) # #args not including 'self'
def_op('BUILD_LIST_FROM_ARG', 203)
jrel_op('JUMP_IF_NOT_DEBUG', 204) # jump over assert statements
+def_op('LOAD_REVDB_VAR', 205) # reverse debugger (syntax example: $5)
del def_op, name_op, jrel_op, jabs_op
diff --git a/lib-python/3/sysconfig.py b/lib-python/3/sysconfig.py
--- a/lib-python/3/sysconfig.py
+++ b/lib-python/3/sysconfig.py
@@ -20,30 +20,30 @@
_INSTALL_SCHEMES = {
'posix_prefix': {
- 'stdlib': '{installed_base}/lib/python{py_version_short}',
- 'platstdlib': '{platbase}/lib/python{py_version_short}',
- 'purelib': '{base}/lib/python{py_version_short}/site-packages',
- 'platlib': '{platbase}/lib/python{py_version_short}/site-packages',
+ 'stdlib': '{installed_base}/lib/{implementation_lower}{py_version_short}',
+ 'platstdlib': '{platbase}/lib/{implementation_lower}{py_version_short}',
+ 'purelib': '{base}/lib/{implementation_lower}{py_version_short}/site-packages',
+ 'platlib': '{platbase}/lib/{implementation_lower}{py_version_short}/site-packages',
'include':
- '{installed_base}/include/python{py_version_short}{abiflags}',
+ '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}',
'platinclude':
- '{installed_platbase}/include/python{py_version_short}{abiflags}',
+ '{installed_platbase}/include/{implementation_lower}{py_version_short}{abiflags}',
'scripts': '{base}/bin',
'data': '{base}',
},
'posix_home': {
- 'stdlib': '{installed_base}/lib/python',
- 'platstdlib': '{base}/lib/python',
- 'purelib': '{base}/lib/python',
- 'platlib': '{base}/lib/python',
- 'include': '{installed_base}/include/python',
- 'platinclude': '{installed_base}/include/python',
+ 'stdlib': '{installed_base}/lib/{implementation_lower}',
+ 'platstdlib': '{base}/lib/{implementation_lower}',
+ 'purelib': '{base}/lib/{implementation_lower}',
+ 'platlib': '{base}/lib/{implementation_lower}',
+ 'include': '{installed_base}/include/{implementation_lower}',
+ 'platinclude': '{installed_base}/include/{implementation_lower}',
'scripts': '{base}/bin',
'data': '{base}',
},
'pypy': {
- 'stdlib': '{installed_base}/lib-python',
- 'platstdlib': '{base}/lib-python',
+ 'stdlib': '{installed_base}/lib-{implementation_lower}',
+ 'platstdlib': '{base}/lib-{implementation_lower}',
'purelib': '{base}/site-packages',
'platlib': '{base}/site-packages',
'include': '{installed_base}/include',
@@ -62,28 +62,28 @@
'data': '{base}',
},
'nt_user': {
- 'stdlib': '{userbase}/Python{py_version_nodot}',
- 'platstdlib': '{userbase}/Python{py_version_nodot}',
- 'purelib': '{userbase}/Python{py_version_nodot}/site-packages',
- 'platlib': '{userbase}/Python{py_version_nodot}/site-packages',
- 'include': '{userbase}/Python{py_version_nodot}/Include',
- 'scripts': '{userbase}/Python{py_version_nodot}/Scripts',
+ 'stdlib': '{userbase}/{implementation}{py_version_nodot}',
+ 'platstdlib': '{userbase}/{implementation}{py_version_nodot}',
+ 'purelib': '{userbase}/{implementation}{py_version_nodot}/site-packages',
+ 'platlib': '{userbase}/{implementation}{py_version_nodot}/site-packages',
+ 'include': '{userbase}/{implementation}{py_version_nodot}/Include',
+ 'scripts': '{userbase}/{implementation}{py_version_nodot}/Scripts',
'data': '{userbase}',
},
'posix_user': {
- 'stdlib': '{userbase}/lib/python{py_version_short}',
- 'platstdlib': '{userbase}/lib/python{py_version_short}',
- 'purelib': '{userbase}/lib/python{py_version_short}/site-packages',
- 'platlib': '{userbase}/lib/python{py_version_short}/site-packages',
- 'include': '{userbase}/include/python{py_version_short}',
+ 'stdlib': '{userbase}/lib/{implementation_lower}{py_version_short}',
+ 'platstdlib': '{userbase}/lib/{implementation_lower}{py_version_short}',
+ 'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages',
+ 'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages',
+ 'include': '{userbase}/include/{implementation_lower}{py_version_short}',
'scripts': '{userbase}/bin',
'data': '{userbase}',
},
'osx_framework_user': {
- 'stdlib': '{userbase}/lib/python',
- 'platstdlib': '{userbase}/lib/python',
- 'purelib': '{userbase}/lib/python/site-packages',
- 'platlib': '{userbase}/lib/python/site-packages',
+ 'stdlib': '{userbase}/lib/{implementation_lower}',
+ 'platstdlib': '{userbase}/lib/{implementation_lower}',
+ 'purelib': '{userbase}/lib/{implementation_lower}/site-packages',
+ 'platlib': '{userbase}/lib/{implementation_lower}/site-packages',
'include': '{userbase}/include',
'scripts': '{userbase}/bin',
'data': '{userbase}',
@@ -106,6 +106,11 @@
_USER_BASE = None
+def _get_implementation():
+ if '__pypy__' in sys.builtin_module_names:
+ return 'PyPy'
+ return 'Python'
+
def _safe_realpath(path):
try:
return realpath(path)
@@ -540,6 +545,8 @@
except AttributeError:
# sys.abiflags may not be defined on all platforms.
_CONFIG_VARS['abiflags'] = ''
+ _CONFIG_VARS['implementation'] = _get_implementation()
+ _CONFIG_VARS['implementation_lower'] = _get_implementation().lower()
if os.name == 'nt':
_init_non_posix(_CONFIG_VARS)
@@ -715,6 +722,8 @@
_print_dict('Paths', get_paths())
print()
_print_dict('Variables', get_config_vars())
+ print
+ _print_dict('User', get_paths('%s_user' % os.name))
if __name__ == '__main__':
diff --git a/lib-python/3/test/test_exceptions.py b/lib-python/3/test/test_exceptions.py
--- a/lib-python/3/test/test_exceptions.py
+++ b/lib-python/3/test/test_exceptions.py
@@ -164,10 +164,10 @@
is_pypy = check_impl_detail(pypy=True)
check('def fact(x):\n\treturn x!\n', 2, 10)
- check('1 +\n', 1, 4 - is_pypy)
- check('def spam():\n print(1)\n print(2)', 3, 0 if is_pypy else 10)
- check('Python = "Python" +', 1, 20 - is_pypy)
- check('Python = "\u1e54\xfd\u0163\u0125\xf2\xf1" +', 1, 20 - is_pypy)
+ check('1 +\n', 1, 4)
+ check('def spam():\n print(1)\n print(2)', 3, 2 if is_pypy else 10)
+ check('Python = "Python" +', 1, 20)
+ check('Python = "\u1e54\xfd\u0163\u0125\xf2\xf1" +', 1, 20)
@cpython_only
def testSettingException(self):
diff --git a/lib-python/3/test/test_fstring.py b/lib-python/3/test/test_fstring.py
--- a/lib-python/3/test/test_fstring.py
+++ b/lib-python/3/test/test_fstring.py
@@ -319,7 +319,7 @@
["f'{3)+(4}'",
])
- self.assertAllRaise(SyntaxError, 'EOL while scanning string literal',
+ self.assertAllRaise(SyntaxError, r'end of line \(EOL\) while scanning string literal',
["f'{\n}'",
])
@@ -741,7 +741,7 @@
self.assertEqual('{d[0]}'.format(d=d), 'integer')
def test_invalid_expressions(self):
- self.assertAllRaise(SyntaxError, 'invalid syntax',
+ self.assertAllRaise(SyntaxError, "closing parenthesis '.' does not match opening parenthesis '.'",
[r"f'{a[4)}'",
r"f'{a(4]}'",
])
diff --git a/lib-python/3/test/test_sysconfig.py b/lib-python/3/test/test_sysconfig.py
--- a/lib-python/3/test/test_sysconfig.py
+++ b/lib-python/3/test/test_sysconfig.py
@@ -4,6 +4,7 @@
import subprocess
import shutil
from copy import copy
+from distutils.spawn import find_executable
from test.support import (run_unittest, TESTFN, unlink, check_warnings,
captured_stdout, impl_detail, import_module,
@@ -297,6 +298,30 @@
self.assertIn(ldflags, ldshared)
+ @unittest.skipIf(sys.platform == "win32", "Does not apply to Windows")
+ def test_cc_values(self):
+ """ CC and CXX should be set for pypy """
+ for var in ["CC", "CXX"]:
+ assert sysconfig.get_config_var(var) is not None
+
+ @unittest.skipIf(not find_executable("gcc"),
+ "Does not apply to machines without gcc installed"
+ )
+ def test_gcc_values(self):
+ """ if gcc is installed on the box, gcc values should be set. """
+ assert "gcc" in sysconfig.get_config_var("CC")
+ assert sysconfig.get_config_var("GNULD") == "yes"
+ assert "gcc" in sysconfig.get_config_var("LDSHARED")
+
+
+ @unittest.skipIf(not find_executable("g++"),
+ "Does not apply to machines without g++ installed"
+ )
+ def test_gplusplus_values(self):
+ """ if g++ is installed on the box, g++ values should be set. """
+ assert "g++" in sysconfig.get_config_var("CXX")
+
+
@unittest.skipUnless(sys.platform == "darwin", "test only relevant on MacOSX")
def test_platform_in_subprocess(self):
my_platform = sysconfig.get_platform()
diff --git a/lib-python/3/test/test_sysconfig_pypy.py b/lib-python/3/test/test_sysconfig_pypy.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3/test/test_sysconfig_pypy.py
@@ -0,0 +1,17 @@
+import os
+import sys
+import unittest
+import site
+
+
+class TestSysConfigPypy(unittest.TestCase):
+ def test_install_schemes(self):
+ # User-site etc. paths should have "pypy" and not "python"
+ # inside them.
+ if site.ENABLE_USER_SITE:
+ parts = site.USER_SITE.lower().split(os.path.sep)
+ assert any(x.startswith('pypy') for x in parts[-2:]), parts
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -422,6 +422,7 @@
RegrTest('test_sys_setprofile.py', core=True),
RegrTest('test_sys_settrace.py', core=True),
RegrTest('test_sysconfig.py'),
+ RegrTest('test_sysconfig_pypy.py'),
RegrTest('test_syslog.py'),
RegrTest('test_tarfile.py'),
RegrTest('test_tcl.py'),
diff --git a/lib_pypy/_cffi_ssl/README.md b/lib_pypy/_cffi_ssl/README.md
--- a/lib_pypy/_cffi_ssl/README.md
+++ b/lib_pypy/_cffi_ssl/README.md
@@ -14,6 +14,8 @@
* ``_cffi_src/openssl/x509_vfy.py`` for issue #2605 (ca4d0c90f5a1)
+* ``_cffi_src/openssl/pypy_win32_extra.py`` for Win32-only functionality like ssl.enum_certificates()
+
# Tests?
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/pypy_win32_extra.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/pypy_win32_extra.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/pypy_win32_extra.py
@@ -0,0 +1,84 @@
+#
+# An extra bit of logic for the Win32-only functionality that is missing from the
+# version from cryptography.
+#
+
+import sys
+
+INCLUDES = """
+#include <Wincrypt.h>
+"""
+
+TYPES = """
+typedef ... *HCERTSTORE;
+typedef ... *HCRYPTPROV_LEGACY;
+
+typedef struct {
+ DWORD dwCertEncodingType;
+ BYTE *pbCertEncoded;
+ DWORD cbCertEncoded;
+ ...;
+} CERT_CONTEXT, *PCCERT_CONTEXT;
+
+typedef struct {
+ DWORD dwCertEncodingType;
+ BYTE *pbCrlEncoded;
+ DWORD cbCrlEncoded;
+ ...;
+} CRL_CONTEXT, *PCCRL_CONTEXT;
+
+typedef struct {
+ DWORD cUsageIdentifier;
+ LPSTR *rgpszUsageIdentifier;
+ ...;
+} CERT_ENHKEY_USAGE, *PCERT_ENHKEY_USAGE;
+"""
+
+FUNCTIONS = """
+HCERTSTORE WINAPI CertOpenStore(
+ LPCSTR lpszStoreProvider,
+ DWORD dwMsgAndCertEncodingType,
+ HCRYPTPROV_LEGACY hCryptProv,
+ DWORD dwFlags,
+ const char *pvPara
+);
+PCCERT_CONTEXT WINAPI CertEnumCertificatesInStore(
+ HCERTSTORE hCertStore,
+ PCCERT_CONTEXT pPrevCertContext
+);
+BOOL WINAPI CertFreeCertificateContext(
+ PCCERT_CONTEXT pCertContext
+);
+BOOL WINAPI CertFreeCRLContext(
+ PCCRL_CONTEXT pCrlContext
+);
+BOOL WINAPI CertCloseStore(
+ HCERTSTORE hCertStore,
+ DWORD dwFlags
+);
+BOOL WINAPI CertGetEnhancedKeyUsage(
+ PCCERT_CONTEXT pCertContext,
+ DWORD dwFlags,
+ PCERT_ENHKEY_USAGE pUsage,
+ DWORD *pcbUsage
+);
+PCCRL_CONTEXT WINAPI CertEnumCRLsInStore(
+ HCERTSTORE hCertStore,
+ PCCRL_CONTEXT pPrevCrlContext
+);
+"""
+
+MACROS = """
+#define CERT_STORE_READONLY_FLAG ...
+#define CERT_SYSTEM_STORE_LOCAL_MACHINE ...
+#define CRYPT_E_NOT_FOUND ...
+#define CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG ...
+#define CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG ...
+#define X509_ASN_ENCODING ...
+#define PKCS_7_ASN_ENCODING ...
+
+static const LPCSTR CERT_STORE_PROV_SYSTEM_A;
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -24,6 +24,7 @@
from enum import IntEnum as _IntEnum
if sys.platform == 'win32':
+ from _cffi_ssl._stdssl.win32_extra import enum_certificates, enum_crls
HAVE_POLL = False
else:
from select import poll, POLLIN, POLLOUT
@@ -219,7 +220,6 @@
if server_hostname:
self.server_hostname = server_hostname.decode('idna', 'strict')
- lib.ERR_get_state()
lib.ERR_clear_error()
self.ssl = ssl = ffi.gc(lib.SSL_new(ctx), lib.SSL_free)
diff --git a/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py b/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py
@@ -0,0 +1,101 @@
+from _pypy_openssl import lib, ffi
+
+
+def enum_certificates(store_name):
+ """Retrieve certificates from Windows' cert store.
+
+store_name may be one of 'CA', 'ROOT' or 'MY'. The system may provide
+more cert storages, too. The function returns a list of (bytes,
+encoding_type, trust) tuples. The encoding_type flag can be interpreted
+with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The trust setting is either
+a set of OIDs or the boolean True.
+ """
+ hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL,
+ lib.CERT_STORE_READONLY_FLAG | lib.CERT_SYSTEM_STORE_LOCAL_MACHINE,
+ bytes(store_name, "ascii"))
+ if hStore == ffi.NULL:
+ raise WindowsError(*ffi.getwinerror())
+
+ result = []
+ pCertCtx = ffi.NULL
+ try:
+ while True:
+ pCertCtx = lib.CertEnumCertificatesInStore(hStore, pCertCtx)
+ if pCertCtx == ffi.NULL:
+ break
+ cert = ffi.buffer(pCertCtx.pbCertEncoded, pCertCtx.cbCertEncoded)[:]
+ enc = certEncodingType(pCertCtx.dwCertEncodingType)
+ keyusage = parseKeyUsage(pCertCtx, lib.CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG)
+ if keyusage is True:
+ keyusage = parseKeyUsage(pCertCtx, lib.CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG)
+ result.append((cert, enc, keyusage))
+ finally:
+ if pCertCtx != ffi.NULL:
+ lib.CertFreeCertificateContext(pCertCtx)
+ if not lib.CertCloseStore(hStore, 0):
+ # This error case might shadow another exception.
+ raise WindowsError(*ffi.getwinerror())
+ return result
+
+
+def enum_crls(store_name):
+ """Retrieve CRLs from Windows' cert store.
+
+store_name may be one of 'CA', 'ROOT' or 'MY'. The system may provide
+more cert storages, too. The function returns a list of (bytes,
+encoding_type) tuples. The encoding_type flag can be interpreted with
+X509_ASN_ENCODING or PKCS_7_ASN_ENCODING."""
+ hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL,
+ lib.CERT_STORE_READONLY_FLAG | lib.CERT_SYSTEM_STORE_LOCAL_MACHINE,
+ bytes(store_name, "ascii"))
+ if hStore == ffi.NULL:
+ raise WindowsError(*ffi.getwinerror())
+
+ result = []
+ pCrlCtx = ffi.NULL
+ try:
+ while True:
+ pCrlCtx = lib.CertEnumCRLsInStore(hStore, pCrlCtx)
+ if pCrlCtx == ffi.NULL:
+ break
+ crl = ffi.buffer(pCrlCtx.pbCrlEncoded, pCrlCtx.cbCrlEncoded)[:]
+ enc = certEncodingType(pCrlCtx.dwCertEncodingType)
+ result.append((crl, enc))
+ finally:
+ if pCrlCtx != ffi.NULL:
+ lib.CertFreeCRLContext(pCrlCtx)
+ if not lib.CertCloseStore(hStore, 0):
+ # This error case might shadow another exception.
+ raise WindowsError(*ffi.getwinerror())
+ return result
+
+
+def certEncodingType(encodingType):
+ if encodingType == lib.X509_ASN_ENCODING:
+ return "x509_asn"
+ if encodingType == lib.PKCS_7_ASN_ENCODING:
+ return "pkcs_7_asn"
+ return encodingType
+
+def parseKeyUsage(pCertCtx, flags):
+ pSize = ffi.new("DWORD *")
+ if not lib.CertGetEnhancedKeyUsage(pCertCtx, flags, ffi.NULL, pSize):
+ error_with_message = ffi.getwinerror()
+ if error_with_message[0] == lib.CRYPT_E_NOT_FOUND:
+ return True
+ raise WindowsError(*error_with_message)
+
+ pUsageMem = ffi.new("char[]", pSize[0])
+ pUsage = ffi.cast("PCERT_ENHKEY_USAGE", pUsageMem)
+ if not lib.CertGetEnhancedKeyUsage(pCertCtx, flags, pUsage, pSize):
+ error_with_message = ffi.getwinerror()
+ if error_with_message[0] == lib.CRYPT_E_NOT_FOUND:
+ return True
+ raise WindowsError(*error_with_message)
+
+ retval = set()
+ for i in range(pUsage.cUsageIdentifier):
+ if pUsage.rgpszUsageIdentifier[i]:
+ oid = ffi.string(pUsage.rgpszUsageIdentifier[i]).decode('ascii')
+ retval.add(oid)
+ return retval
diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -81,8 +81,11 @@
def _CData_output(self, resarray, base=None, index=-1):
from _rawffi.alt import types
# If a char_p or unichar_p is received, skip the string interpretation
- if base._ffiargtype != types.Pointer(types.char_p) and \
- base._ffiargtype != types.Pointer(types.unichar_p):
+ try:
+ deref = type(base)._deref_ffiargtype()
+ except AttributeError:
+ deref = None
+ if deref != types.char_p and deref != types.unichar_p:
# this seems to be a string if we're array of char, surprise!
from ctypes import c_char, c_wchar
if self._type_ is c_char:
@@ -127,6 +130,12 @@
value = self(*value)
return _CDataMeta.from_param(self, value)
+ def _build_ffiargtype(self):
+ return _ffi.types.Pointer(self._type_.get_ffi_argtype())
+
+ def _deref_ffiargtype(self):
+ return self._type_.get_ffi_argtype()
+
def array_get_slice_params(self, index):
if hasattr(self, '_length_'):
start, stop, step = index.indices(self._length_)
@@ -254,6 +263,5 @@
_type_ = base
)
cls = ArrayMeta(name, (Array,), tpdict)
- cls._ffiargtype = _ffi.types.Pointer(base.get_ffi_argtype())
ARRAY_CACHE[key] = cls
return cls
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -49,10 +49,13 @@
else:
return self.from_param(as_parameter)
+ def _build_ffiargtype(self):
+ return _shape_to_ffi_type(self._ffiargshape_)
+
def get_ffi_argtype(self):
if self._ffiargtype:
return self._ffiargtype
- self._ffiargtype = _shape_to_ffi_type(self._ffiargshape_)
+ self._ffiargtype = self._build_ffiargtype()
return self._ffiargtype
def _CData_output(self, resbuffer, base=None, index=-1):
diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -70,7 +70,12 @@
self._ffiarray = ffiarray
self.__init__ = __init__
self._type_ = TP
- self._ffiargtype = _ffi.types.Pointer(TP.get_ffi_argtype())
+
+ def _build_ffiargtype(self):
+ return _ffi.types.Pointer(self._type_.get_ffi_argtype())
+
+ def _deref_ffiargtype(self):
+ return self._type_.get_ffi_argtype()
from_address = cdata_from_address
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -160,6 +160,10 @@
raise AttributeError("_fields_ is final")
if self in [f[1] for f in value]:
raise AttributeError("Structure or union cannot contain itself")
+ if self._ffiargtype is not None:
+ raise NotImplementedError("Too late to set _fields_: we already "
+ "said to libffi that the structure type %s is opaque"
+ % (self,))
names_and_fields(
self,
value, self.__bases__[0],
diff --git a/lib_pypy/_ssl/__init__.py b/lib_pypy/_ssl/__init__.py
--- a/lib_pypy/_ssl/__init__.py
+++ b/lib_pypy/_ssl/__init__.py
@@ -16,12 +16,14 @@
RAND_egd = builtinify(RAND_egd)
import sys
-if sys.platform == "win32" and 'enum_certificates' not in globals():
- def enum_certificates(*args, **kwds):
- import warnings
- warnings.warn("ssl.enum_certificates() is not implemented")
- return []
- def enum_crls(*args, **kwds):
- import warnings
- warnings.warn("ssl.enum_crls() is not implemented")
- return []
+if sys.platform == "win32":
+ if 'enum_certificates' not in globals():
+ def enum_certificates(*args, **kwds):
+ import warnings
+ warnings.warn("ssl.enum_certificates() is not implemented")
+ return []
+ if 'enum_crls' not in globals():
+ def enum_crls(*args, **kwds):
+ import warnings
+ warnings.warn("ssl.enum_crls() is not implemented")
+ return []
diff --git a/lib_pypy/_ssl_build.py b/lib_pypy/_ssl_build.py
--- a/lib_pypy/_ssl_build.py
+++ b/lib_pypy/_ssl_build.py
@@ -5,6 +5,11 @@
from _cffi_ssl._cffi_src.build_openssl import (build_ffi_for_binding,
_get_openssl_libraries, extra_link_args, compiler_type)
+if sys.platform == "win32":
+ pypy_win32_extra = ["pypy_win32_extra"]
+else:
+ pypy_win32_extra = []
+
ffi = build_ffi_for_binding(
module_name="_pypy_openssl",
module_prefix="_cffi_src.openssl.",
@@ -44,10 +49,10 @@
"x509_vfy",
"pkcs7",
"callbacks",
- ],
+ ] + pypy_win32_extra,
libraries=_get_openssl_libraries(sys.platform),
extra_link_args=extra_link_args(compiler_type()),
)
if __name__ == '__main__':
- ffi.compile()
+ ffi.compile(verbose=True)
diff --git a/lib_pypy/_sysconfigdata.py b/lib_pypy/_sysconfigdata.py
--- a/lib_pypy/_sysconfigdata.py
+++ b/lib_pypy/_sysconfigdata.py
@@ -1,10 +1,50 @@
import _imp
+import os
+import sys
+from distutils.spawn import find_executable
so_ext = _imp.extension_suffixes()[0]
+
build_time_vars = {
- "EXT_SUFFIX": so_ext,
- "SHLIB_SUFFIX": so_ext,
"SOABI": '-'.join(so_ext.split('.')[1].split('-')[:2]),
- "SO": so_ext # deprecated in Python 3, for backward compatibility
+ "SO": so_ext, # deprecated in Python 3, for backward compatibility
+ 'CC': "cc -pthread",
+ 'CXX': "c++ -pthread",
+ 'OPT': "-DNDEBUG -O2",
+ 'CFLAGS': "-DNDEBUG -O2",
+ 'CCSHARED': "-fPIC",
+ 'LDSHARED': "cc -pthread -shared",
+ 'EXT_SUFFIX': so_ext,
+ 'SHLIB_SUFFIX': ".so",
+ 'AR': "ar",
+ 'ARFLAGS': "rc",
+ 'EXE': "",
+ 'LIBDIR': os.path.join(sys.prefix, 'lib'),
+ 'VERSION': sys.version[:3]
}
+
+if find_executable("gcc"):
+ build_time_vars.update({
+ "CC": "gcc -pthread",
+ "GNULD": "yes",
+ "LDSHARED": "gcc -pthread -shared",
+ })
+ if find_executable("g++"):
+ build_time_vars["CXX"] = "g++ -pthread"
+
+if sys.platform[:6] == "darwin":
+ import platform
+ if platform.machine() == 'i386':
+ if platform.architecture()[0] == '32bit':
+ arch = 'i386'
+ else:
+ arch = 'x86_64'
+ else:
+ # just a guess
+ arch = platform.machine()
+ build_time_vars['LDSHARED'] += ' -undefined dynamic_lookup'
+ build_time_vars['CC'] += ' -arch %s' % (arch,)
+ if "CXX" in build_time_vars:
+ build_time_vars['CXX'] += ' -arch %s' % (arch,)
+
diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py
--- a/lib_pypy/_winapi.py
+++ b/lib_pypy/_winapi.py
@@ -98,9 +98,11 @@
def GetOverlappedResult(self, wait):
transferred = _ffi.new('DWORD[1]', [0])
res = _kernel32.GetOverlappedResult(self.handle, self.overlapped, transferred, wait != 0)
- if not res:
- res = GetLastError()
- if res in (ERROR_SUCCESS, ERROR_MORE_DATA, ERROR_OPERATION_ABORTED):
+ if res:
+ err = ERROR_SUCCESS
+ else:
+ err = GetLastError()
+ if err in (ERROR_SUCCESS, ERROR_MORE_DATA, ERROR_OPERATION_ABORTED):
self.completed = 1
self.pending = 0
elif res == ERROR_IO_INCOMPLETE:
@@ -133,7 +135,7 @@
assert success == 0
err = _kernel32.GetLastError()
if err == ERROR_IO_PENDING:
- overlapped[0].pending = 1
+ ov.pending = 1
elif err == ERROR_PIPE_CONNECTED:
_kernel32.SetEvent(ov.overlapped[0].hEvent)
else:
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: cffi
-Version: 1.11.5
+Version: 1.12.0
Summary: Foreign Function Interface for Python calling C code.
Home-page: http://cffi.readthedocs.org
Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
from .api import FFI
from .error import CDefError, FFIError, VerificationError, VerificationMissing
-__version__ = "1.11.5"
-__version_info__ = (1, 11, 5)
+__version__ = "1.12.0"
+__version_info__ = (1, 12, 0)
# The verifier module file names are based on the CRC32 of a string that
# contains the following version number. It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -221,7 +221,7 @@
if (f != NULL && f != Py_None) {
PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
- "\ncompiled with cffi version: 1.11.5"
+ "\ncompiled with cffi version: 1.12.0"
"\n_cffi_backend module: ", f);
modules = PyImport_GetModuleDict();
mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -96,18 +96,21 @@
self.CData, self.CType = backend._get_types()
self.buffer = backend.buffer
- def cdef(self, csource, override=False, packed=False):
+ def cdef(self, csource, override=False, packed=False, pack=None):
"""Parse the given C source. This registers all declared functions,
types, and global variables. The functions and global variables can
then be accessed via either 'ffi.dlopen()' or 'ffi.verify()'.
The types can be used in 'ffi.new()' and other functions.
If 'packed' is specified as True, all structs declared inside this
cdef are packed, i.e. laid out without any field alignment at all.
+ Alternatively, 'pack' can be a small integer, and requests for
+ alignment greater than that are ignored (pack=1 is equivalent to
+ packed=True).
"""
- self._cdef(csource, override=override, packed=packed)
+ self._cdef(csource, override=override, packed=packed, pack=pack)
- def embedding_api(self, csource, packed=False):
- self._cdef(csource, packed=packed, dllexport=True)
+ def embedding_api(self, csource, packed=False, pack=None):
+ self._cdef(csource, packed=packed, pack=pack, dllexport=True)
if self._embedding is None:
self._embedding = ''
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -730,7 +730,8 @@
return self._new_struct_or_union('union', name, ctypes.Union)
def complete_struct_or_union(self, CTypesStructOrUnion, fields, tp,
- totalsize=-1, totalalignment=-1, sflags=0):
+ totalsize=-1, totalalignment=-1, sflags=0,
+ pack=0):
if totalsize >= 0 or totalalignment >= 0:
raise NotImplementedError("the ctypes backend of CFFI does not support "
"structures completed by verify(); please "
@@ -751,6 +752,8 @@
bfield_types[fname] = Ellipsis
if sflags & 8:
struct_or_union._pack_ = 1
+ elif pack:
+ struct_or_union._pack_ = pack
struct_or_union._fields_ = cfields
CTypesStructOrUnion._bfield_types = bfield_types
#
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -306,11 +306,25 @@
msg = 'parse error\n%s' % (msg,)
raise CDefError(msg)
- def parse(self, csource, override=False, packed=False, dllexport=False):
+ def parse(self, csource, override=False, packed=False, pack=None,
+ dllexport=False):
+ if packed:
+ if packed != True:
+ raise ValueError("'packed' should be False or True; use "
+ "'pack' to give another value")
+ if pack:
+ raise ValueError("cannot give both 'pack' and 'packed'")
+ pack = 1
+ elif pack:
+ if pack & (pack - 1):
+ raise ValueError("'pack' must be a power of two, not %r" %
+ (pack,))
+ else:
+ pack = 0
prev_options = self._options
try:
self._options = {'override': override,
- 'packed': packed,
+ 'packed': pack,
'dllexport': dllexport}
self._internal_parse(csource)
finally:
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -342,7 +342,7 @@
fixedlayout = None
completed = 0
partial = False
- packed = False
+ packed = 0
def __init__(self, name, fldnames, fldtypes, fldbitsize, fldquals=None):
self.name = name
@@ -414,11 +414,14 @@
fldtypes = [tp.get_cached_btype(ffi, finishlist)
for tp in self.fldtypes]
lst = list(zip(self.fldnames, fldtypes, self.fldbitsize))
- sflags = 0
+ extra_flags = ()
if self.packed:
- sflags = 8 # SF_PACKED
+ if self.packed == 1:
+ extra_flags = (8,) # SF_PACKED
+ else:
+ extra_flags = (0, self.packed)
ffi._backend.complete_struct_or_union(BType, lst, self,
- -1, -1, sflags)
+ -1, -1, *extra_flags)
#
else:
fldtypes = []
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -893,6 +893,12 @@
else:
flags.append("_CFFI_F_CHECK_FIELDS")
if tp.packed:
+ if tp.packed > 1:
+ raise NotImplementedError(
+ "%r is declared with 'pack=%r'; only 0 or 1 are "
+ "supported in API mode (try to use \"...;\", which "
+ "does not require a 'pack' declaration)" %
+ (tp, tp.packed))
flags.append("_CFFI_F_PACKED")
else:
flags.append("_CFFI_F_EXTERNAL")
diff --git a/lib_pypy/grp.py b/lib_pypy/grp.py
--- a/lib_pypy/grp.py
+++ b/lib_pypy/grp.py
@@ -5,6 +5,8 @@
import os
from _pwdgrp_cffi import ffi, lib
import _structseq
+import thread
+_lock = thread.allocate_lock()
try: from __pypy__ import builtinify
except ImportError: builtinify = lambda f: f
@@ -33,31 +35,34 @@
@builtinify
def getgrgid(gid):
- res = lib.getgrgid(gid)
- if not res:
- # XXX maybe check error eventually
- raise KeyError(gid)
- return _group_from_gstruct(res)
+ with _lock:
+ res = lib.getgrgid(gid)
+ if not res:
+ # XXX maybe check error eventually
+ raise KeyError(gid)
+ return _group_from_gstruct(res)
@builtinify
def getgrnam(name):
if not isinstance(name, str):
raise TypeError("expected string")
- res = lib.getgrnam(os.fsencode(name))
- if not res:
- raise KeyError("'getgrnam(): name not found: %s'" % name)
- return _group_from_gstruct(res)
+ with _lock:
+ res = lib.getgrnam(os.fsencode(name))
+ if not res:
+ raise KeyError("'getgrnam(): name not found: %s'" % name)
+ return _group_from_gstruct(res)
@builtinify
def getgrall():
- lib.setgrent()
lst = []
- while 1:
- p = lib.getgrent()
- if not p:
- break
- lst.append(_group_from_gstruct(p))
- lib.endgrent()
+ with _lock:
+ lib.setgrent()
+ while 1:
+ p = lib.getgrent()
+ if not p:
+ break
+ lst.append(_group_from_gstruct(p))
+ lib.endgrent()
return lst
__all__ = ('struct_group', 'getgrgid', 'getgrnam', 'getgrall')
diff --git a/lib_pypy/pwd.py b/lib_pypy/pwd.py
--- a/lib_pypy/pwd.py
+++ b/lib_pypy/pwd.py
@@ -12,6 +12,8 @@
from _pwdgrp_cffi import ffi, lib
import _structseq
+import thread
+_lock = thread.allocate_lock()
try: from __pypy__ import builtinify
except ImportError: builtinify = lambda f: f
@@ -54,10 +56,11 @@
Return the password database entry for the given numeric user ID.
See pwd.__doc__ for more on password database entries.
"""
- pw = lib.getpwuid(uid)
- if not pw:
- raise KeyError("getpwuid(): uid not found: %s" % uid)
- return _mkpwent(pw)
+ with _lock:
+ pw = lib.getpwuid(uid)
+ if not pw:
+ raise KeyError("getpwuid(): uid not found: %s" % uid)
+ return _mkpwent(pw)
@builtinify
def getpwnam(name):
@@ -70,10 +73,11 @@
if not isinstance(name, basestring):
raise TypeError("expected string")
name = str(name)
- pw = lib.getpwnam(name)
- if not pw:
- raise KeyError("getpwname(): name not found: %s" % name)
- return _mkpwent(pw)
+ with _lock:
+ pw = lib.getpwnam(name)
+ if not pw:
+ raise KeyError("getpwname(): name not found: %s" % name)
+ return _mkpwent(pw)
@builtinify
def getpwall():
@@ -83,13 +87,14 @@
See pwd.__doc__ for more on password database entries.
"""
users = []
- lib.setpwent()
- while True:
- pw = lib.getpwent()
- if not pw:
- break
- users.append(_mkpwent(pw))
- lib.endpwent()
+ with _lock:
+ lib.setpwent()
+ while True:
+ pw = lib.getpwent()
+ if not pw:
+ break
+ users.append(_mkpwent(pw))
+ lib.endpwent()
return users
__all__ = ('struct_passwd', 'getpwuid', 'getpwnam', 'getpwall')
diff --git a/lib_pypy/pyrepl/simple_interact.py b/lib_pypy/pyrepl/simple_interact.py
--- a/lib_pypy/pyrepl/simple_interact.py
+++ b/lib_pypy/pyrepl/simple_interact.py
@@ -66,6 +66,10 @@
while 1:
try:
+ try:
+ sys.stdout.flush()
+ except:
+ pass
ps1 = getattr(sys, 'ps1', '>>> ')
ps2 = getattr(sys, 'ps2', '... ')
try:
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -39,18 +39,14 @@
"thread", "itertools", "pyexpat", "cpyext", "array",
"binascii", "_multiprocessing", '_warnings', "_collections",
"_multibytecodec", "_continuation", "_cffi_backend",
- "_csv", "_pypyjson", "_posixsubprocess", # "_cppyy", "micronumpy"
+ "_csv", "_pypyjson", "_posixsubprocess", "_cppyy", # "micronumpy",
"_jitlog",
])
-from rpython.jit.backend import detect_cpu
-try:
- if detect_cpu.autodetect().startswith('x86'):
- if not sys.platform.startswith('openbsd'):
- working_modules.add('_vmprof')
- working_modules.add('faulthandler')
-except detect_cpu.ProcessorAutodetectError:
- pass
+import rpython.rlib.rvmprof.cintf
+if rpython.rlib.rvmprof.cintf.IS_SUPPORTED:
+ working_modules.add('_vmprof')
+ working_modules.add('faulthandler')
translation_modules = default_modules.copy()
translation_modules.update([
@@ -61,6 +57,11 @@
"termios", "_minimal_curses",
])
+reverse_debugger_disable_modules = set([
+ "_continuation", "_vmprof", "_multiprocessing",
+ "micronumpy",
+ ])
+
# XXX this should move somewhere else, maybe to platform ("is this posixish"
# check or something)
if sys.platform == "win32":
@@ -301,6 +302,9 @@
modules = working_modules.copy()
if config.translation.sandbox:
modules = default_modules
+ if config.translation.reverse_debugger:
+ for mod in reverse_debugger_disable_modules:
+ setattr(config.objspace.usemodules, mod, False)
# ignore names from 'essential_modules', notably 'exceptions', which
# may not be present in config.objspace.usemodules at all
modules = [name for name in modules if name not in essential_modules]
@@ -319,3 +323,4 @@
parser = to_optparse(config) #, useoptions=["translation.*"])
option, args = parser.parse_args()
print config
+ print working_modules
diff --git a/pypy/doc/architecture.rst b/pypy/doc/architecture.rst
--- a/pypy/doc/architecture.rst
+++ b/pypy/doc/architecture.rst
@@ -73,3 +73,63 @@
This division between bytecode evaluator and object space gives a lot of
flexibility. One can plug in different :doc:`object spaces <objspace>` to get
different or enriched behaviours of the Python objects.
+
+Layers
+------
+
+RPython
+~~~~~~~
+:ref:`RPython <rpython:language>` is the language in which we write interpreters.
+Not the entire PyPy project is written in RPython, only the parts that are
+compiled in the translation process. The interesting point is that RPython
+has no parser, it's compiled from the live python objects, which makes it
+possible to do all kinds of metaprogramming during import time. In short,
+Python is a meta programming language for RPython.
+
+The RPython standard library is to be found in the ``rlib`` subdirectory.
+
+Consult `Getting Started with RPython`_ for further reading
+
+Translation
+~~~~~~~~~~~
+The translation toolchain - this is the part that takes care of translating
+RPython to flow graphs and then to C. There is more in the
+:doc:`architecture <architecture>` document written about it.
+
+It lives in the ``rpython`` directory: ``flowspace``, ``annotator``
+and ``rtyper``.
+
+PyPy Interpreter
+~~~~~~~~~~~~~~~~
+This is in the ``pypy`` directory. ``pypy/interpreter`` is a standard
+interpreter for Python written in RPython. The fact that it is
+RPython is not apparent at first. Built-in modules are written in
+``pypy/module/*``. Some modules that CPython implements in C are
+simply written in pure Python; they are in the top-level ``lib_pypy``
+directory. The standard library of Python (with a few changes to
+accomodate PyPy) is in ``lib-python``.
+
+JIT Compiler
+~~~~~~~~~~~~
+:ref:`Just-in-Time Compiler (JIT) <rpython:jit>`: we have a tracing JIT that traces the
+interpreter written in RPython, rather than the user program that it
+interprets. As a result it applies to any interpreter, i.e. any
+language. But getting it to work correctly is not trivial: it
+requires a small number of precise "hints" and possibly some small
+refactorings of the interpreter. The JIT itself also has several
+almost-independent parts: the tracer itself in ``rpython/jit/metainterp``, the
+optimizer in ``rpython/jit/metainterp/optimizer`` that optimizes a list of
+residual operations, and the backend in ``rpython/jit/backend/<machine-name>``
+that turns it into machine code. Writing a new backend is a
+traditional way to get into the project.
+
+Garbage Collectors
+~~~~~~~~~~~~~~~~~~
+Garbage Collectors (GC): as you may notice if you are used to CPython's
+C code, there are no ``Py_INCREF/Py_DECREF`` equivalents in RPython code.
+:ref:`rpython:garbage-collection` is inserted
+during translation. Moreover, this is not reference counting; it is a real
+GC written as more RPython code. The best one we have so far is in
+``rpython/memory/gc/incminimark.py``.
+
+.. _`Getting started with RPython`: http://rpython.readthedocs.org/en/latest/getting-started.html
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -267,14 +267,14 @@
* PyPy 2.5.1 or earlier: normal users would see permission errors.
Installers need to run ``pypy -c "import gdbm"`` and other similar
commands at install time; the exact list is in
- :source:`pypy/tool/release/package.py <package.py>`. Users
+ :source:`pypy/tool/release/package.py`. Users
seeing a broken installation of PyPy can fix it after-the-fact if they
have sudo rights, by running once e.g. ``sudo pypy -c "import gdbm``.
* PyPy 2.6 and later: anyone would get ``ImportError: no module named
_gdbm_cffi``. Installers need to run ``pypy _gdbm_build.py`` in the
``lib_pypy`` directory during the installation process (plus others;
- see the exact list in :source:`pypy/tool/release/package.py <package.py>`).
+ see the exact list in :source:`pypy/tool/release/package.py`).
Users seeing a broken
installation of PyPy can fix it after-the-fact, by running ``pypy
/path/to/lib_pypy/_gdbm_build.py``. This command produces a file
diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst
--- a/pypy/doc/coding-guide.rst
+++ b/pypy/doc/coding-guide.rst
@@ -539,7 +539,7 @@
hg help branch
-.. _official wiki: http://mercurial.selenic.com/wiki/Branch
+.. _official wiki: https://www.mercurial-scm.org/wiki/
.. _using-development-tracker:
@@ -547,15 +547,7 @@
Using the development bug/feature tracker
-----------------------------------------
-We have a `development tracker`_, based on Richard Jones'
-`roundup`_ application. You can file bugs,
-feature requests or see what's going on
-for the next milestone, both from an E-Mail and from a
-web interface.
-
-.. _development tracker: https://bugs.pypy.org/
-.. _roundup: http://roundup.sourceforge.net/
-
+We use bitbucket for :source:`issues` tracking and :source:`pull-requests`.
.. _testing:
diff --git a/pypy/doc/commandline_ref.rst b/pypy/doc/commandline_ref.rst
--- a/pypy/doc/commandline_ref.rst
+++ b/pypy/doc/commandline_ref.rst
@@ -8,3 +8,4 @@
:maxdepth: 1
man/pypy.1.rst
+ man/pypy3.1.rst
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -66,9 +66,9 @@
# built documents.
#
# The short X.Y version.
-version = '5.8'
+version = '6.0'
# The full version, including alpha/beta/rc tags.
-release = '5.8.0'
+release = '6.0.0'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
diff --git a/pypy/doc/config/objspace.disable_entrypoints.txt b/pypy/doc/config/objspace.disable_entrypoints.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.fstrings.txt b/pypy/doc/config/objspace.fstrings.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.hash.txt b/pypy/doc/config/objspace.hash.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.usemodules._cppyy.txt b/pypy/doc/config/objspace.usemodules._cppyy.txt
new file mode 100644
--- /dev/null
+++ b/pypy/doc/config/objspace.usemodules._cppyy.txt
@@ -0,0 +1,1 @@
+The internal backend for cppyy
diff --git a/pypy/doc/config/objspace.usemodules._frozen_importlib.txt b/pypy/doc/config/objspace.usemodules._frozen_importlib.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.usemodules._jitlog.txt b/pypy/doc/config/objspace.usemodules._jitlog.txt
new file mode 100644
diff --git a/pypy/doc/config/objspace.usemodules._rawffi.txt b/pypy/doc/config/objspace.usemodules._rawffi.txt
--- a/pypy/doc/config/objspace.usemodules._rawffi.txt
+++ b/pypy/doc/config/objspace.usemodules._rawffi.txt
@@ -1,3 +1,3 @@
-An experimental module providing very low-level interface to
+A module providing very low-level interface to
C-level libraries, for use when implementing ctypes, not
-intended for a direct use at all.
\ No newline at end of file
+intended for a direct use at all.
diff --git a/pypy/doc/config/objspace.usemodules.cpyext.txt b/pypy/doc/config/objspace.usemodules.cpyext.txt
--- a/pypy/doc/config/objspace.usemodules.cpyext.txt
+++ b/pypy/doc/config/objspace.usemodules.cpyext.txt
@@ -1,1 +1,1 @@
-Use (experimental) cpyext module, that tries to load and run CPython extension modules
+Use cpyext module to load and run CPython extension modules
diff --git a/pypy/doc/config/objspace.usemodules.faulthandler.txt b/pypy/doc/config/objspace.usemodules.faulthandler.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.backendopt.replace_we_are_jitted.txt b/pypy/doc/config/translation.backendopt.replace_we_are_jitted.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.jit_opencoder_model.txt b/pypy/doc/config/translation.jit_opencoder_model.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.keepgoing.txt b/pypy/doc/config/translation.keepgoing.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.libname.txt b/pypy/doc/config/translation.libname.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.lto.txt b/pypy/doc/config/translation.lto.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.profoptargs.txt b/pypy/doc/config/translation.profoptargs.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.reverse_debugger.txt b/pypy/doc/config/translation.reverse_debugger.txt
new file mode 100644
diff --git a/pypy/doc/config/translation.split_gc_address_space.txt b/pypy/doc/config/translation.split_gc_address_space.txt
new file mode 100644
diff --git a/pypy/doc/contributing.rst b/pypy/doc/contributing.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/contributing.rst
@@ -0,0 +1,472 @@
+Contributing Guidelines
+===========================
+
+.. contents::
+
+PyPy is a very large project that has a reputation of being hard to dive into.
+Some of this fame is warranted, some of it is purely accidental. There are three
+important lessons that everyone willing to contribute should learn:
+
+* PyPy has layers. There are many pieces of architecture that are very well
+ separated from each other. More about this below, but often the manifestation
+ of this is that things are at a different layer than you would expect them
+ to be. For example if you are looking for the JIT implementation, you will
+ not find it in the implementation of the Python programming language.
+
+* Because of the above, we are very serious about Test Driven Development.
+ It's not only what we believe in, but also that PyPy's architecture is
+ working very well with TDD in mind and not so well without it. Often
+ development means progressing in an unrelated corner, one unittest
+ at a time; and then flipping a giant switch, bringing it all together.
+ (It generally works out of the box. If it doesn't, then we didn't
+ write enough unit tests.) It's worth repeating - PyPy's
+ approach is great if you do TDD, and not so great otherwise.
+
+* PyPy uses an entirely different set of tools - most of them included
+ in the PyPy repository. There is no Makefile, nor autoconf. More below.
+
+The first thing to remember is that PyPy project is very different than most
+projects out there. It's also different from a classic compiler project,
+so academic courses about compilers often don't apply or lead in the wrong
+direction. However, if you want to understand how designing & building a runtime
+works in the real world then this is a great project!
+
+Getting involved
+^^^^^^^^^^^^^^^^
+
+PyPy employs a relatively standard open-source development process. You are
+encouraged as a first step to join our `pypy-dev mailing list`_ and IRC channel,
+details of which can be found in our :ref:`contact <contact>` section. The folks
+there are very friendly, and can point you in the right direction.
+
+We give out commit rights usually fairly liberally, so if you want to do something
+with PyPy, you can become a committer. We also run frequent coding sprints which
+are separately announced and often happen around Python conferences such as
+EuroPython or PyCon. Upcoming events are usually announced on `the blog`_.
+
+Further Reading: :ref:`Contact <contact>`
+
+.. _the blog: http://morepypy.blogspot.com
+.. _pypy-dev mailing list: http://mail.python.org/mailman/listinfo/pypy-dev
+
+
+Your first contribution
+^^^^^^^^^^^^^^^^^^^^^^^
+
+The first and most important rule how **not** to contribute to PyPy is
+"just hacking a feature". This won't work, and you'll find your PR will typically
+require a lot of re-work. There are a few reasons why not:
+
+* build times are large
+* PyPy has very thick layer separation
+* context of the cPython runtime is often required
+
+Instead, reach out on the dev mailing list or the IRC channel, and we're more
+than happy to help! :)
+
+Some ideas for first contributions are:
+
+* Documentation - this will give you an understanding of the pypy architecture
+* Test failures - find a failing test in the `nightly builds`_, and fix it
+* Missing language features - these are listed in our `issue tracker`_
+
+.. _nightly builds: http://buildbot.pypy.org/nightly/
+.. _issue tracker: https://bitbucket.org/pypy/pypy/issues
+
+Source Control
+--------------
+
+PyPy development is based a typical fork/pull request based workflow, centered
+around Mercurial (hg), hosted on Bitbucket. If you have not used this workflow
+before, a good introduction can be found here:
+
+ https://www.atlassian.com/git/tutorials/comparing-workflows/forking-workflow
+
+The cycle for a new PyPy contributor goes typically like this:
+
+Fork & Clone
+------------
+
+* Make an account on bitbucket_.
+
+* Go to https://bitbucket.org/pypy/pypy/ and click "fork" (left
+ icons). You get a fork of the repository, e.g. in
+ `https://bitbucket.org/yourname/pypy/`.
+
+* Clone your new repo (i.e. the fork) to your local machine with the command
+ ``hg clone ssh://hg@bitbucket.org/yourname/pypy``. It is a very slow
+ operation but only ever needs to be done once. See also
+ http://pypy.org/download.html#building-from-source .
+ If you already cloned
+ ``https://bitbucket.org/pypy/pypy`` before, even if some time ago,
+ then you can reuse the same clone by editing the file ``.hg/hgrc`` in
+ your clone to contain the line ``default =
+ ssh://hg@bitbucket.org/yourname/pypy``, and then do ``hg pull && hg
+ up``. If you already have such a clone but don't want to change it,
+ you can clone that copy with ``hg clone /path/to/other/copy``, and
+ then edit ``.hg/hgrc`` as above and do ``hg pull && hg up``.
+
+* Now you have a complete copy of the PyPy repo. Make a branch
+ with a command like ``hg branch name_of_your_branch``.
+
+Edit
+----
+
+* Edit things. Use ``hg diff`` to see what you changed. Use ``hg add``
+ to make Mercurial aware of new files you added, e.g. new test files.
+ Use ``hg status`` to see if there are such files. Write and run tests!
+ (See the rest of this page.)
+
+* Commit regularly with ``hg commit``. A one-line commit message is
+ fine. We love to have tons of commits; make one as soon as you have
+ some progress, even if it is only some new test that doesn't pass yet,
+ or fixing things even if not all tests pass. Step by step, you are
+ building the history of your changes, which is the point of a version
+ control system. (There are commands like ``hg log`` and ``hg up``
+ that you should read about later, to learn how to navigate this
+ history.)
+
+* The commits stay on your machine until you do ``hg push`` to "push"
+ them back to the repo named in the file ``.hg/hgrc``. Repos are
+ basically just collections of commits (a commit is also called a
+ changeset): there is one repo per url, plus one for each local copy on
+ each local machine. The commands ``hg push`` and ``hg pull`` copy
+ commits around, with the goal that all repos in question end up with
+ the exact same set of commits. By opposition, ``hg up`` only updates
+ the "working copy" by reading the local repository, i.e. it makes the
+ files that you see correspond to the latest (or any other) commit
+ locally present.
+
+* You should push often; there is no real reason not to. Remember that
+ even if they are pushed, with the setup above, the commits are (1)
+ only in ``bitbucket.org/yourname/pypy``, and (2) in the branch you
+ named. Yes, they are publicly visible, but don't worry about someone
+ walking around the thousands of repos on bitbucket saying "hah, look
+ at the bad coding style of that guy". Try to get into the mindset
+ that your work is not secret and it's fine that way. We might not
+ accept it as is for PyPy, asking you instead to improve some things,
+ but we are not going to judge you.
+
+Pull Request
+------------
+
+* The final step is to open a pull request, so that we know that you'd
+ like to merge that branch back to the original ``pypy/pypy`` repo.
+ This can also be done several times if you have interesting
+ intermediate states, but if you get there, then we're likely to
+ proceed to the next stage, which is...
+
+* Get a regular account for pushing directly to
+ ``bitbucket.org/pypy/pypy`` (just ask and you'll get it, basically).
+ Once you have it you can rewrite your file ``.hg/hgrc`` to contain
+ ``default = ssh://hg@bitbucket.org/pypy/pypy``. Your changes will
+ then be pushed directly to the official repo, but (if you follow these
+ rules) they are still on a branch, and we can still review the
+ branches you want to merge.
+
+* If you get closer to the regular day-to-day development, you'll notice
+ that we generally push small changes as one or a few commits directly
+ to the branch ``default``. Also, we often collaborate even if we are
+ on other branches, which do not really "belong" to anyone. At this
+ point you'll need ``hg merge`` and learn how to resolve conflicts that
+ sometimes occur when two people try to push different commits in
+ parallel on the same branch. But it is likely an issue for later ``:-)``
+
+.. _bitbucket: https://bitbucket.org/
+
+
+Architecture
+^^^^^^^^^^^^
+
+PyPy has layers. Just like ogres or onions. Those layers help us keep the
+respective parts separated enough to be worked on independently and make the
+complexity manageable. This is, again, just a sanity requirement for such
+a complex project. For example writing a new optimization for the JIT usually
+does **not** involve touching a Python interpreter at all or the JIT assembler
+backend or the garbage collector. Instead it requires writing small tests in
+``rpython/jit/metainterp/optimizeopt/test/test_*`` and fixing files there.
+After that, you can just compile PyPy and things should just work.
+
+Further Reading: :doc:`architecture <architecture>`
+
+Where to start?
+---------------
+
+PyPy is made from parts that are relatively independent of each other.
+You should start looking at the part that attracts you most (all paths are
+relative to the PyPy top level directory). You may look at our
+:doc:`directory reference <dir-reference>` or start off at one of the following
+points:
+
+* :source:`pypy/interpreter` contains the bytecode interpreter: bytecode dispatcher
+ in :source:`pypy/interpreter/pyopcode.py`, frame and code objects in
+ :source:`pypy/interpreter/eval.py` and :source:`pypy/interpreter/pyframe.py`,
+ function objects and argument passing in :source:`pypy/interpreter/function.py`
+ and :source:`pypy/interpreter/argument.py`, the object space interface
+ definition in :source:`pypy/interpreter/baseobjspace.py`, modules in
+ :source:`pypy/interpreter/module.py` and :source:`pypy/interpreter/mixedmodule.py`.
+ Core types supporting the bytecode interpreter are defined in
+ :source:`pypy/interpreter/typedef.py`.
+
+* :source:`pypy/interpreter/pyparser` contains a recursive descent parser,
+ and grammar files that allow it to parse the syntax of various Python
+ versions. Once the grammar has been processed, the parser can be
+ translated by the above machinery into efficient code.
+
+* :source:`pypy/interpreter/astcompiler` contains the compiler. This
+ contains a modified version of the compiler package from CPython
+ that fixes some bugs and is translatable.
+
+* :source:`pypy/objspace/std` contains the
+ :ref:`Standard object space <standard-object-space>`. The main file
+ is :source:`pypy/objspace/std/objspace.py`. For each type, the file
+ ``xxxobject.py`` contains the implementation for objects of type ``xxx``,
+ as a first approximation. (Some types have multiple implementations.)
+
+Building
+^^^^^^^^
+
+For building PyPy, we recommend installing a pre-built PyPy first (see
+:doc:`install`). It is possible to build PyPy with CPython, but it will take a
+lot longer to run -- depending on your architecture, between two and three
+times as long.
+
+Further Reading: :doc:`Build <build>`
+
+Coding Guide
+------------
+
+As well as the usual pep8 and formatting standards, there are a number of
+naming conventions and coding styles that are important to understand before
+browsing the source.
+
+Further Reading: :doc:`Coding Guide <coding-guide>`
+
+Testing
+^^^^^^^
+
+Test driven development
+-----------------------
+
+Instead, we practice a lot of test driven development. This is partly because
+of very high quality requirements for compilers and partly because there is
+simply no other way to get around such complex project, that will keep you sane.
+There are probably people out there who are smart enough not to need it, we're
+not one of those. You may consider familiarizing yourself with `pytest`_,
+since this is a tool we use for tests.
+This leads to the next issue:
+
+.. _pytest: http://pytest.org/
+
+py.test and the py lib
+----------------------
+
+The `py.test testing tool`_ drives all our testing needs.
+
+We use the `py library`_ for filesystem path manipulations, terminal
+writing, logging and some other support functionality.
+
+You don't necessarily need to install these two libraries because
+we also ship them inlined in the PyPy source tree.
+
+.. _py library: http://pylib.readthedocs.org/
+
+Running PyPy's unit tests
+-------------------------
+
+PyPy development always was and is still thoroughly test-driven.
+We use the flexible `py.test testing tool`_ which you can `install independently
+<http://pytest.org/latest/getting-started.html#getstarted>`_ and use for other projects.
+
+The PyPy source tree comes with an inlined version of ``py.test``
+which you can invoke by typing::
+
+ python pytest.py -h
+
+This is usually equivalent to using an installed version::
+
+ py.test -h
+
+If you encounter problems with the installed version
+make sure you have the correct version installed which
+you can find out with the ``--version`` switch.
+
+You will need the `build requirements`_ to run tests successfully, since many of
+them compile little pieces of PyPy and then run the tests inside that minimal
+interpreter. The `cpyext` tests also require `pycparser`, and many tests build
+cases with `hypothesis`.
+
+Now on to running some tests. PyPy has many different test directories
+and you can use shell completion to point at directories or files::
+
+ py.test pypy/interpreter/test/test_pyframe.py
+
+ # or for running tests of a whole subdirectory
+ py.test pypy/interpreter/
+
+See `py.test usage and invocations`_ for some more generic info
+on how you can run tests.
+
+Beware trying to run "all" pypy tests by pointing to the root
+directory or even the top level subdirectory ``pypy``. It takes
+hours and uses huge amounts of RAM and is not recommended.
+
+To run CPython regression tests you can point to the ``lib-python``
+directory::
+
+ py.test lib-python/2.7/test/test_datetime.py
+
+This will usually take a long time because this will run
+the PyPy Python interpreter on top of CPython. On the plus
+side, it's usually still faster than doing a full translation
+and running the regression test with the translated PyPy Python
+interpreter.
+
+.. _py.test testing tool: http://pytest.org
+.. _py.test usage and invocations: http://pytest.org/latest/usage.html#usage
+.. _`build requirements`: build.html#install-build-time-dependencies
+
+Testing After Translation
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+While the usual invocation of `pytest` translates a piece of RPython code and
+runs it, we have a test extension to run tests without translation, directly
+on the host python. This is very convenient for modules such as `cpyext`, to
+compare and contrast test results between CPython and PyPy. Untranslated tests
+are invoked by using the `-A` or `--runappdirect` option to `pytest`::
+
+ python2 pytest.py -A pypy/module/cpyext/test
+
+where `python2` can be either `python2` or `pypy2`. On the `py3` branch, the
+collection phase must be run with `python2` so untranslated tests are run
+with::
+
+ cpython2 pytest.py -A pypy/module/cpyext/test --python=path/to/pypy3
+
+
+Tooling & Utilities
+^^^^^^^^^^^^^^^^^^^
+
+If you are interested in the inner workings of the PyPy Python interpreter,
+there are some features of the untranslated Python interpreter that allow you
+to introspect its internals.
+
+
+Interpreter-level console
+-------------------------
+
+To start interpreting Python with PyPy, install a C compiler that is
+supported by distutils and use Python 2.7 or greater to run PyPy::
+
+ cd pypy
+ python bin/pyinteractive.py
+
+After a few seconds (remember: this is running on top of CPython), you should
+be at the PyPy prompt, which is the same as the Python prompt, but with an
+extra ">".
+
+If you press
+<Ctrl-C> on the console you enter the interpreter-level console, a
+usual CPython console. You can then access internal objects of PyPy
+(e.g. the :ref:`object space <objspace>`) and any variables you have created on the PyPy
+prompt with the prefix ``w_``::
+
+ >>>> a = 123
+ >>>> <Ctrl-C>
+ *** Entering interpreter-level console ***
+ >>> w_a
+ W_IntObject(123)
+
+The mechanism works in both directions. If you define a variable with the ``w_`` prefix on the interpreter-level, you will see it on the app-level::
+
+ >>> w_l = space.newlist([space.wrap(1), space.wrap("abc")])
+ >>> <Ctrl-D>
+ *** Leaving interpreter-level console ***
+
+ KeyboardInterrupt
+ >>>> l
+ [1, 'abc']
+
+Note that the prompt of the interpreter-level console is only '>>>' since
+it runs on CPython level. If you want to return to PyPy, press <Ctrl-D> (under
+Linux) or <Ctrl-Z>, <Enter> (under Windows).
+
+Also note that not all modules are available by default in this mode (for
+example: ``_continuation`` needed by ``greenlet``) , you may need to use one of
+``--withmod-...`` command line options.
+
+You may be interested in reading more about the distinction between
+:ref:`interpreter-level and app-level <interpreter-level>`.
+
+pyinteractive.py options
+------------------------
+
+To list the PyPy interpreter command line options, type::
+
+ cd pypy
+ python bin/pyinteractive.py --help
+
+pyinteractive.py supports most of the options that CPython supports too (in addition to a
+large amount of options that can be used to customize pyinteractive.py).
+As an example of using PyPy from the command line, you could type::
+
+ python pyinteractive.py --withmod-time -c "from test import pystone; pystone.main(10)"
+
+Alternatively, as with regular Python, you can simply give a
+script name on the command line::
+
+ python pyinteractive.py --withmod-time ../../lib-python/2.7/test/pystone.py 10
+
+The ``--withmod-xxx`` option enables the built-in module ``xxx``. By
+default almost none of them are, because initializing them takes time.
+If you want anyway to enable all built-in modules, you can use
+``--allworkingmodules``.
+
+See our :doc:`configuration sections <config/index>` for details about what all the commandline
+options do.
+
+
+.. _trace example:
+
+Tracing bytecode and operations on objects
+------------------------------------------
+
+You can use a simple tracing mode to monitor the interpretation of
+bytecodes. To enable it, set ``__pytrace__ = 1`` on the interactive
+PyPy console::
+
+ >>>> __pytrace__ = 1
+ Tracing enabled
+ >>>> x = 5
+ <module>: LOAD_CONST 0 (5)
+ <module>: STORE_NAME 0 (x)
+ <module>: LOAD_CONST 1 (None)
+ <module>: RETURN_VALUE 0
+ >>>> x
+ <module>: LOAD_NAME 0 (x)
+ <module>: PRINT_EXPR 0
+ 5
+ <module>: LOAD_CONST 0 (None)
+ <module>: RETURN_VALUE 0
+ >>>>
+
+
+Demos
+^^^^^
+
+The `example-interpreter`_ repository contains an example interpreter
+written using the RPython translation toolchain.
+
+.. _example-interpreter: https://bitbucket.org/pypy/example-interpreter
+
+
+graphviz & pygame for flow graph viewing (highly recommended)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+graphviz and pygame are both necessary if you want to look at generated flow
+graphs:
+
+ graphviz: http://www.graphviz.org/Download.php
+
+ pygame: http://www.pygame.org/download.shtml
+
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -214,6 +214,7 @@
Lukas Vacek
Omer Katz
Jacek Generowicz
+ Tomasz Dziopa
Sylvain Thenault
Jakub Stasiak
Andrew Dalke
@@ -274,6 +275,7 @@
Yury V. Zaytsev
florinpapa
Anders Sigfridsson
+ Matt Jackson
Nikolay Zinov
rafalgalczynski at gmail.com
Joshua Gilbert
diff --git a/pypy/doc/discussion/ctypes-implementation.rst b/pypy/doc/discussion/ctypes-implementation.rst
--- a/pypy/doc/discussion/ctypes-implementation.rst
+++ b/pypy/doc/discussion/ctypes-implementation.rst
@@ -141,28 +141,3 @@
.. _pyglet: http://pyglet.org/
More information about the pypy-commit
mailing list