[pypy-commit] pypy default: merge heads
arigo
pypy.commits at gmail.com
Thu Feb 22 02:05:20 EST 2018
Author: Armin Rigo <arigo at tunes.org>
Branch:
Changeset: r93866:8f992014885b
Date: 2018-02-22 08:04 +0100
http://bitbucket.org/pypy/pypy/changeset/8f992014885b/
Log: merge heads
diff too long, truncating to 2000 out of 4036 lines
diff --git a/pypy/module/test_lib_pypy/pyrepl/__init__.py b/extra_tests/test_pyrepl/__init__.py
rename from pypy/module/test_lib_pypy/pyrepl/__init__.py
rename to extra_tests/test_pyrepl/__init__.py
--- a/pypy/module/test_lib_pypy/pyrepl/__init__.py
+++ b/extra_tests/test_pyrepl/__init__.py
@@ -1,3 +1,1 @@
-import sys
-import lib_pypy.pyrepl
-sys.modules['pyrepl'] = sys.modules['lib_pypy.pyrepl']
+
diff --git a/pypy/module/test_lib_pypy/pyrepl/infrastructure.py b/extra_tests/test_pyrepl/infrastructure.py
rename from pypy/module/test_lib_pypy/pyrepl/infrastructure.py
rename to extra_tests/test_pyrepl/infrastructure.py
diff --git a/pypy/module/test_lib_pypy/pyrepl/test_basic.py b/extra_tests/test_pyrepl/test_basic.py
rename from pypy/module/test_lib_pypy/pyrepl/test_basic.py
rename to extra_tests/test_pyrepl/test_basic.py
diff --git a/pypy/module/test_lib_pypy/pyrepl/test_bugs.py b/extra_tests/test_pyrepl/test_bugs.py
rename from pypy/module/test_lib_pypy/pyrepl/test_bugs.py
rename to extra_tests/test_pyrepl/test_bugs.py
diff --git a/extra_tests/test_pyrepl/test_functional.py b/extra_tests/test_pyrepl/test_functional.py
new file mode 100644
--- /dev/null
+++ b/extra_tests/test_pyrepl/test_functional.py
@@ -0,0 +1,28 @@
+# Copyright 2000-2007 Michael Hudson-Doyle <micahel at gmail.com>
+# Maciek Fijalkowski
+# License: MIT
+# some functional tests, to see if this is really working
+
+import pytest
+import sys
+
+
+ at pytest.fixture()
+def child():
+ try:
+ import pexpect
+ except ImportError:
+ pytest.skip("no pexpect module")
+ except SyntaxError:
+ pytest.skip('pexpect wont work on py3k')
+ child = pexpect.spawn(sys.executable, ['-S'], timeout=10)
+ child.logfile = sys.stdout
+ child.sendline('from pyrepl.python_reader import main')
+ child.sendline('main()')
+ return child
+
+
+def test_basic(child):
+ child.sendline('a = 3')
+ child.sendline('a')
+ child.expect('3')
diff --git a/pypy/module/test_lib_pypy/pyrepl/test_keymap.py b/extra_tests/test_pyrepl/test_keymap.py
rename from pypy/module/test_lib_pypy/pyrepl/test_keymap.py
rename to extra_tests/test_pyrepl/test_keymap.py
diff --git a/pypy/module/test_lib_pypy/pyrepl/test_reader.py b/extra_tests/test_pyrepl/test_reader.py
rename from pypy/module/test_lib_pypy/pyrepl/test_reader.py
rename to extra_tests/test_pyrepl/test_reader.py
diff --git a/pypy/module/test_lib_pypy/pyrepl/test_readline.py b/extra_tests/test_pyrepl/test_readline.py
rename from pypy/module/test_lib_pypy/pyrepl/test_readline.py
rename to extra_tests/test_pyrepl/test_readline.py
diff --git a/pypy/module/test_lib_pypy/pyrepl/test_wishes.py b/extra_tests/test_pyrepl/test_wishes.py
rename from pypy/module/test_lib_pypy/pyrepl/test_wishes.py
rename to extra_tests/test_pyrepl/test_wishes.py
diff --git a/get_externals.py b/get_externals.py
new file mode 100644
--- /dev/null
+++ b/get_externals.py
@@ -0,0 +1,69 @@
+'''Get external dependencies for building PyPy
+they will end up in the platform.host().basepath, something like repo-root/external
+'''
+
+from __future__ import print_function
+
+import argparse
+import os
+import zipfile
+from subprocess import Popen, PIPE
+from rpython.translator.platform import host
+
+def runcmd(cmd, verbose):
+ stdout = stderr = ''
+ report = False
+ try:
+ p = Popen(cmd, stdout=PIPE, stderr=PIPE)
+ stdout, stderr = p.communicate()
+ if p.wait() != 0 or verbose:
+ report = True
+ except Exception as e:
+ stderr = str(e) + '\n' + stderr
+ report = True
+ if report:
+ print('running "%s" returned\n%s\n%s' % (' '.join(cmd), stdout, stderr))
+ if stderr:
+ raise RuntimeError(stderr)
+
+def checkout_repo(dest='externals', org='pypy', branch='default', verbose=False):
+ url = 'https://bitbucket.org/{}/externals'.format(org)
+ if not os.path.exists(dest):
+ cmd = ['hg','clone',url,dest]
+ runcmd(cmd, verbose)
+ cmd = ['hg','-R', dest, 'update',branch]
+ runcmd(cmd, verbose)
+
+def extract_zip(externals_dir, zip_path):
+ with zipfile.ZipFile(os.fspath(zip_path)) as zf:
+ zf.extractall(os.fspath(externals_dir))
+ return externals_dir / zf.namelist()[0].split('/')[0]
+
+def parse_args():
+ p = argparse.ArgumentParser()
+ p.add_argument('-v', '--verbose', action='store_true')
+ p.add_argument('-O', '--organization',
+ help='Organization owning the deps repos', default='pypy')
+ p.add_argument('-e', '--externals', default=host.externals,
+ help='directory in which to store dependencies',
+ )
+ p.add_argument('-b', '--branch', default=host.externals_branch,
+ help='branch to check out',
+ )
+ p.add_argument('-p', '--platform', default=None,
+ help='someday support cross-compilation, ignore for now',
+ )
+ return p.parse_args()
+
+
+def main():
+ args = parse_args()
+ checkout_repo(
+ dest=args.externals,
+ org=args.organization,
+ branch=args.branch,
+ verbose=args.verbose,
+ )
+
+if __name__ == '__main__':
+ main()
diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst
--- a/pypy/doc/gc_info.rst
+++ b/pypy/doc/gc_info.rst
@@ -1,17 +1,137 @@
-Garbage collector configuration
-===============================
+Garbage collector documentation and configuration
+=================================================
+
+
+Incminimark
+-----------
+
+PyPy's default garbage collector is called incminimark - it's an incremental,
+generational moving collector. Here we hope to explain a bit how it works
+and how it can be tuned to suit the workload.
+
+Incminimark first allocates objects in so called *nursery* - place for young
+objects, where allocation is very cheap, being just a pointer bump. The nursery
+size is a very crucial variable - depending on your workload (one or many
+processes) and cache sizes you might want to experiment with it via
+*PYPY_GC_NURSERY* environment variable. When the nursery is full, there is
+performed a minor collection. Freed objects are no longer referencable and
+just die, just by not being referenced any more; on the other hand, objects
+found to still be alive must survive and are copied from the nursery
+to the old generation. Either to arenas, which are collections
+of objects of the same size, or directly allocated with malloc if they're
+larger. (A third category, the very large objects, are initially allocated
+outside the nursery and never move.)
+
+Since Incminimark is an incremental GC, the major collection is incremental,
+meaning there should not be any pauses longer than 1ms.
+
+
+Fragmentation
+-------------
+
+Before we discuss issues of "fragmentation", we need a bit of precision.
+There are two kinds of related but distinct issues:
+
+* If the program allocates a lot of memory, and then frees it all by
+ dropping all references to it, then we might expect to see the RSS
+ to drop. (RSS = Resident Set Size on Linux, as seen by "top"; it is an
+ approximation of the actual memory usage from the OS's point of view.)
+ This might not occur: the RSS may remain at its highest value. This
+ issue is more precisely caused by the process not returning "free"
+ memory to the OS. We call this case "unreturned memory".
+
+* After doing the above, if the RSS didn't go down, then at least future
+ allocations should not cause the RSS to grow more. That is, the process
+ should reuse unreturned memory as long as it has got some left. If this
+ does not occur, the RSS grows even larger and we have real fragmentation
+ issues.
+
+
+gc.get_stats
+------------
+
+There is a special function in the ``gc`` module called
+``get_stats(memory_pressure=False)``.
+
+``memory_pressure`` controls whether or not to report memory pressure from
+objects allocated outside of the GC, which requires walking the entire heap,
+so it's disabled by default due to its cost. Enable it when debugging
+mysterious memory disappearance.
+
+Example call looks like that::
+
+ >>> gc.get_stats(True)
+ Total memory consumed:
+ GC used: 4.2MB (peak: 4.2MB)
+ in arenas: 763.7kB
+ rawmalloced: 383.1kB
+ nursery: 3.1MB
+ raw assembler used: 0.0kB
+ memory pressure: 0.0kB
+ -----------------------------
+ Total: 4.2MB
+
+ Total memory allocated:
+ GC allocated: 4.5MB (peak: 4.5MB)
+ in arenas: 763.7kB
+ rawmalloced: 383.1kB
+ nursery: 3.1MB
+ raw assembler allocated: 0.0kB
+ memory pressure: 0.0kB
+ -----------------------------
+ Total: 4.5MB
+
+In this particular case, which is just at startup, GC consumes relatively
+little memory and there is even less unused, but allocated memory. In case
+there is a lot of unreturned memory or actual fragmentation, the "allocated"
+can be much higher than "used". Generally speaking, "peak" will more closely
+resemble the actual memory consumed as reported by RSS. Indeed, returning
+memory to the OS is a hard and not solved problem. In PyPy, it occurs only if
+an arena is entirely free---a contiguous block of 64 pages of 4 or 8 KB each.
+It is also rare for the "rawmalloced" category, at least for common system
+implementations of ``malloc()``.
+
+The details of various fields:
+
+* GC in arenas - small old objects held in arenas. If the amount "allocated"
+ is much higher than the amount "used", we have unreturned memory. It is
+ possible but unlikely that we have internal fragmentation here. However,
+ this unreturned memory cannot be reused for any ``malloc()``, including the
+ memory from the "rawmalloced" section.
+
+* GC rawmalloced - large objects allocated with malloc. This is gives the
+ current (first block of text) and peak (second block of text) memory
+ allocated with ``malloc()``. The amount of unreturned memory or
+ fragmentation caused by ``malloc()`` cannot easily be reported. Usually
+ you can guess there is some if the RSS is much larger than the total
+ memory reported for "GC allocated", but do keep in mind that this total
+ does not include malloc'ed memory not known to PyPy's GC at all. If you
+ guess there is some, consider using `jemalloc`_ as opposed to system malloc.
+
+.. _`jemalloc`: http://jemalloc.net/
+
+* nursery - amount of memory allocated for nursery, fixed at startup,
+ controlled via an environment variable
+
+* raw assembler allocated - amount of assembler memory that JIT feels
+ responsible for
+
+* memory pressure, if asked for - amount of memory we think got allocated
+ via external malloc (eg loading cert store in SSL contexts) that is kept
+ alive by GC objects, but not accounted in the GC
+
.. _minimark-environment-variables:
-Minimark
---------
+Environment variables
+---------------------
PyPy's default ``incminimark`` garbage collector is configurable through
several environment variables:
``PYPY_GC_NURSERY``
The nursery size.
- Defaults to 1/2 of your cache or ``4M``.
+ Defaults to 1/2 of your last-level cache, or ``4M`` if unknown.
Small values (like 1 or 1KB) are useful for debugging.
``PYPY_GC_NURSERY_DEBUG``
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -40,3 +40,7 @@
.. branch: memory-accounting
Improve way to describe memory
+
+.. branch: msvc14
+
+Allow compilaiton with Visual Studio 2017 compiler suite on windows
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -39,10 +39,24 @@
.. _Microsoft Visual C++ Compiler for Python 2.7: https://www.microsoft.com/en-us/download/details.aspx?id=44266
+Installing "Build Tools for Visual Studio 2017" (for Python 3)
+--------------------------------------------------------------
+
+As documented in the CPython Wiki_, CPython now recommends Visual C++ version
+14.0. A compact version of the compiler suite can be obtained from Microsoft_
+downloads, search the page for "Build Tools for Visual Studio 2017".
+
+You will also need to install the the `Windows SDK`_ in order to use the
+`mt.exe` mainfest compiler.
+
+.. _Wiki: https://wiki.python.org/moin/WindowsCompilers
+.. _Microsoft: https://www.visualstudio.com/downloads
+.. _`Windows SDK`: https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk
+
Translating PyPy with Visual Studio
-----------------------------------
-We routinely test translation using v9, also known as Visual Studio 2008.
+We routinely test translation of PyPy 2.7 using v9 and PyPy 3 with vc14.
Other configurations may work as well.
The translation scripts will set up the appropriate environment variables
@@ -82,8 +96,8 @@
.. _build instructions: http://pypy.org/download.html#building-from-source
-Setting Up Visual Studio for building SSL in Python3
-----------------------------------------------------
+Setting Up Visual Studio 9.0 for building SSL in Python3
+--------------------------------------------------------
On Python3, the ``ssl`` module is based on ``cffi``, and requires a build step after
translation. However ``distutils`` does not support the Micorosft-provided Visual C
@@ -132,243 +146,14 @@
Installing external packages
----------------------------
-On Windows, there is no standard place where to download, build and
-install third-party libraries. We recommend installing them in the parent
-directory of the pypy checkout. For example, if you installed pypy in
-``d:\pypy\trunk\`` (This directory contains a README file), the base
-directory is ``d:\pypy``. You must then set the
-INCLUDE, LIB and PATH (for DLLs) environment variables appropriately.
+We uses a `repository` parallel to pypy to hold binary compiled versions of the
+build dependencies for windows. As part of the `rpython` setup stage, environment
+variables will be set to use these dependencies. The repository has a README
+file on how to replicate, and a branch for each supported platform. You may run
+ the `get_externals.py` utility to checkout the proper branch for your platform
+and PyPy version.
-
-Abridged method (using Visual Studio 2008)
-------------------------------------------
-
-Download the versions of all the external packages from
-https://bitbucket.org/pypy/pypy/downloads/local_59.zip
-(for post-5.8 builds) with sha256 checksum
-``6344230e90ab7a9cb84efbae1ba22051cdeeb40a31823e0808545b705aba8911``
-https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip
-(to reproduce 5.8 builds) with sha256 checksum
-``fbe769bf3a4ab6f5a8b0a05b61930fc7f37da2a9a85a8f609cf5a9bad06e2554`` or
-https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip
-(for 2.4 release and later) or
-https://bitbucket.org/pypy/pypy/downloads/local.zip
-(for pre-2.4 versions)
-Then expand it into the base directory (base_dir) and modify your environment
-to reflect this::
-
- set PATH=<base_dir>\bin;%PATH%
- set INCLUDE=<base_dir>\include;%INCLUDE%
- set LIB=<base_dir>\lib;%LIB%
-
-Now you should be good to go. If you choose this method, you do not need
-to download/build anything else.
-
-Nonabridged method (building from scratch)
-------------------------------------------
-
-If you want to, you can rebuild everything from scratch by continuing.
-
-
-The Boehm garbage collector
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-This library is needed if you plan to use the ``--gc=boehm`` translation
-option (this is the default at some optimization levels like ``-O1``,
-but unneeded for high-performance translations like ``-O2``).
-You may get it at
-http://hboehm.info/gc/gc_source/gc-7.1.tar.gz
-
-Versions 7.0 and 7.1 are known to work; the 6.x series won't work with
-RPython. Unpack this folder in the base directory.
-The default GC_abort(...) function in misc.c will try to open a MessageBox.
-You may want to disable this with the following patch::
-
- --- a/misc.c Sun Apr 20 14:08:27 2014 +0300
- +++ b/misc.c Sun Apr 20 14:08:37 2014 +0300
- @@ -1058,7 +1058,7 @@
- #ifndef PCR
- void GC_abort(const char *msg)
- {
- -# if defined(MSWIN32)
- +# if 0 && defined(MSWIN32)
- (void) MessageBoxA(NULL, msg, "Fatal error in gc", MB_ICONERROR|MB_OK);
- # else
- GC_err_printf("%s\n", msg);
-
-Then open a command prompt::
-
- cd gc-7.1
- nmake -f NT_THREADS_MAKEFILE
- copy Release\gc.dll <somewhere in the PATH>
-
-
-The zlib compression library
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Download http://www.gzip.org/zlib/zlib-1.2.11.tar.gz and extract it in
-the base directory. Then compile::
-
- cd zlib-1.2.11
- nmake -f win32\Makefile.msc
- copy zlib.lib <somewhere in LIB>
- copy zlib.h zconf.h <somewhere in INCLUDE>
- copy zlib1.dll <in PATH> # (needed for tests via ll2ctypes)
-
-
-The bz2 compression library
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Get the same version of bz2 used by python and compile as a static library::
-
- svn export http://svn.python.org/projects/external/bzip2-1.0.6
- cd bzip2-1.0.6
- nmake -f makefile.msc
- copy libbz2.lib <somewhere in LIB>
- copy bzlib.h <somewhere in INCLUDE>
-
-
-The sqlite3 database library
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-PyPy uses cffi to interact with sqlite3.dll. Only the dll is needed, the cffi
-wrapper is compiled when the module is imported for the first time.
-The sqlite3.dll should be version 3.8.11 for CPython2.7 compatablility.
-
-
-The expat XML parser
-~~~~~~~~~~~~~~~~~~~~
-
-CPython compiles expat from source as part of the build. PyPy uses the same
-code base, but expects to link to a static lib of expat. Here are instructions
-to reproduce the static lib in version 2.2.4.
-
-Download the source code of expat: https://github.com/libexpat/libexpat.
-``git checkout`` the proper tag, in this case ``R_2_2_4``. Run
-``vcvars.bat`` to set up the visual compiler tools, and CD into the source
-directory. Create a file ``stdbool.h`` with the content
-
-.. code-block:: c
-
- #pragma once
-
- #define false 0
- #define true 1
-
- #define bool int
-
-and put it in a place on the ``INCLUDE`` path, or create it in the local
-directory and add ``.`` to the ``INCLUDE`` path::
-
- SET INCLUDE=%INCLUDE%;.
-
-Then compile all the ``*.c`` file into ``*.obj``::
-
- cl.exe /nologo /MD /O2 *c /c
- rem for debug
- cl.exe /nologo /MD /O0 /Ob0 /Zi *c /c
-
-You may need to move some variable declarations to the beginning of the
-function, to be compliant with C89 standard. Here is the diff for version 2.2.4
-
-.. code-block:: diff
-
- diff --git a/expat/lib/xmltok.c b/expat/lib/xmltok.c
- index 007aed0..a2dcaad 100644
- --- a/expat/lib/xmltok.c
- +++ b/expat/lib/xmltok.c
- @@ -399,19 +399,21 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc),
- /* Avoid copying partial characters (due to limited space). */
- const ptrdiff_t bytesAvailable = fromLim - *fromP;
- const ptrdiff_t bytesStorable = toLim - *toP;
- + const char * fromLimBefore;
- + ptrdiff_t bytesToCopy;
- if (bytesAvailable > bytesStorable) {
- fromLim = *fromP + bytesStorable;
- output_exhausted = true;
- }
-
- /* Avoid copying partial characters (from incomplete input). */
- - const char * const fromLimBefore = fromLim;
- + fromLimBefore = fromLim;
- align_limit_to_full_utf8_characters(*fromP, &fromLim);
- if (fromLim < fromLimBefore) {
- input_incomplete = true;
- }
-
- - const ptrdiff_t bytesToCopy = fromLim - *fromP;
- + bytesToCopy = fromLim - *fromP;
- memcpy((void *)*toP, (const void *)*fromP, (size_t)bytesToCopy);
- *fromP += bytesToCopy;
- *toP += bytesToCopy;
-
-
-Create ``libexpat.lib`` (for translation) and ``libexpat.dll`` (for tests)::
-
- cl /LD *.obj libexpat.def /Felibexpat.dll
- rem for debug
- rem cl /LDd /Zi *.obj libexpat.def /Felibexpat.dll
-
- rem this will override the export library created in the step above
- rem but tests do not need the export library, they load the dll dynamically
- lib *.obj /out:libexpat.lib
-
-Then, copy
-
-- ``libexpat.lib`` into LIB
-- both ``lib\expat.h`` and ``lib\expat_external.h`` in INCLUDE
-- ``libexpat.dll`` into PATH
-
-
-The OpenSSL library
-~~~~~~~~~~~~~~~~~~~
-
-OpenSSL needs a Perl interpreter to configure its makefile. You may
-use the one distributed by ActiveState, or the one from cygwin.::
-
- svn export http://svn.python.org/projects/external/openssl-1.0.2k
- cd openssl-1.0.2k
- perl Configure VC-WIN32 no-idea no-mdc2
- ms\do_ms.bat
- nmake -f ms\nt.mak install
- copy out32\*.lib <somewhere in LIB>
- xcopy /S include\openssl <somewhere in INCLUDE>
-
-For tests you will also need the dlls::
- nmake -f ms\ntdll.mak install
- copy out32dll\*.dll <somewhere in PATH>
-
-TkInter module support
-~~~~~~~~~~~~~~~~~~~~~~
-
-Note that much of this is taken from the cpython build process.
-Tkinter is imported via cffi, so the module is optional. To recreate the tcltk
-directory found for the release script, create the dlls, libs, headers and
-runtime by running::
-
- svn export http://svn.python.org/projects/external/tcl-8.5.2.1 tcl85
- svn export http://svn.python.org/projects/external/tk-8.5.2.0 tk85
- cd tcl85\win
- nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=0 INSTALLDIR=..\..\tcltk clean all
- nmake -f makefile.vc DEBUG=0 INSTALLDIR=..\..\tcltk install
- cd ..\..\tk85\win
- nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 clean all
- nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 install
- copy ..\..\tcltk\bin\* <somewhere in PATH>
- copy ..\..\tcltk\lib\*.lib <somewhere in LIB>
- xcopy /S ..\..\tcltk\include <somewhere in INCLUDE>
-
-The lzma compression library
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Python 3.3 ship with CFFI wrappers for the lzma library, which can be
-downloaded from this site http://tukaani.org/xz. Python 3.3-3.5 use version
-5.0.5, a prebuilt version can be downloaded from
-http://tukaani.org/xz/xz-5.0.5-windows.zip, check the signature
-http://tukaani.org/xz/xz-5.0.5-windows.zip.sig
-
-Then copy the headers to the include directory, rename ``liblzma.a`` to
-``lzma.lib`` and copy it to the lib directory
-
+.. _repository: https://bitbucket.org/pypy/external
Using the mingw compiler
------------------------
diff --git a/pypy/interpreter/test/test_unicodehelper.py b/pypy/interpreter/test/test_unicodehelper.py
--- a/pypy/interpreter/test/test_unicodehelper.py
+++ b/pypy/interpreter/test/test_unicodehelper.py
@@ -1,5 +1,6 @@
import pytest
import struct
+import sys
from pypy.interpreter.unicodehelper import (
encode_utf8, decode_utf8, unicode_encode_utf_32_be)
@@ -26,7 +27,10 @@
got = decode_utf8(space, "\xed\xa0\x80\xed\xb0\x80")
assert map(ord, got) == [0xd800, 0xdc00]
got = decode_utf8(space, "\xf0\x90\x80\x80")
- assert map(ord, got) == [0x10000]
+ if sys.maxunicode > 65535:
+ assert map(ord, got) == [0x10000]
+ else:
+ assert map(ord, got) == [55296, 56320]
@pytest.mark.parametrize('unich', [u"\ud800", u"\udc80"])
def test_utf32_surrogates(unich):
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -87,7 +87,6 @@
'get_hidden_tb' : 'interp_magic.get_hidden_tb',
'lookup_special' : 'interp_magic.lookup_special',
'do_what_I_mean' : 'interp_magic.do_what_I_mean',
- 'validate_fd' : 'interp_magic.validate_fd',
'resizelist_hint' : 'interp_magic.resizelist_hint',
'newlist_hint' : 'interp_magic.newlist_hint',
'add_memory_pressure' : 'interp_magic.add_memory_pressure',
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -107,14 +107,6 @@
raise oefmt(space.w_TypeError, "expecting dict or list or set object")
return space.newtext(name)
-
- at unwrap_spec(fd='c_int')
-def validate_fd(space, fd):
- try:
- rposix.validate_fd(fd)
- except OSError as e:
- raise wrap_oserror(space, e)
-
def get_console_cp(space):
from rpython.rlib import rwin32 # Windows only
return space.newtuple([
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -538,11 +538,16 @@
assert '\x00'.decode('unicode-internal', 'ignore') == ''
def test_backslashreplace(self):
+ import sys
sin = u"a\xac\u1234\u20ac\u8000\U0010ffff"
- expected = "a\\xac\\u1234\\u20ac\\u8000\\U0010ffff"
- assert sin.encode('ascii', 'backslashreplace') == expected
- expected = "a\xac\\u1234\xa4\\u8000\\U0010ffff"
- assert sin.encode("iso-8859-15", "backslashreplace") == expected
+ if sys.maxunicode > 65535:
+ expected_ascii = "a\\xac\\u1234\\u20ac\\u8000\\U0010ffff"
+ expected_8859 = "a\xac\\u1234\xa4\\u8000\\U0010ffff"
+ else:
+ expected_ascii = "a\\xac\\u1234\\u20ac\\u8000\\udbff\\udfff"
+ expected_8859 = "a\xac\\u1234\xa4\\u8000\\udbff\\udfff"
+ assert sin.encode('ascii', 'backslashreplace') == expected_ascii
+ assert sin.encode("iso-8859-15", "backslashreplace") == expected_8859
def test_badhandler(self):
import codecs
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -30,7 +30,7 @@
from pypy.module.__builtin__.interp_classobj import W_ClassObject
from pypy.module.micronumpy.base import W_NDimArray
from rpython.rlib.entrypoint import entrypoint_lowlevel
-from rpython.rlib.rposix import is_valid_fd, validate_fd
+from rpython.rlib.rposix import FdValidator
from rpython.rlib.unroll import unrolling_iterable
from rpython.rlib.objectmodel import specialize
from pypy.module import exceptions
@@ -96,25 +96,24 @@
dash = ''
def fclose(fp):
- if not is_valid_fd(c_fileno(fp)):
+ try:
+ with FdValidator(c_fileno(fp)):
+ return c_fclose(fp)
+ except IOError:
return -1
- return c_fclose(fp)
def fwrite(buf, sz, n, fp):
- validate_fd(c_fileno(fp))
- return c_fwrite(buf, sz, n, fp)
+ with FdValidator(c_fileno(fp)):
+ return c_fwrite(buf, sz, n, fp)
def fread(buf, sz, n, fp):
- validate_fd(c_fileno(fp))
- return c_fread(buf, sz, n, fp)
+ with FdValidator(c_fileno(fp)):
+ return c_fread(buf, sz, n, fp)
_feof = rffi.llexternal('feof', [FILEP], rffi.INT)
def feof(fp):
- validate_fd(c_fileno(fp))
- return _feof(fp)
-
-def is_valid_fp(fp):
- return is_valid_fd(c_fileno(fp))
+ with FdValidator(c_fileno(fp)):
+ return _feof(fp)
pypy_decl = 'pypy_decl.h'
udir.join(pypy_decl).write("/* Will be filled later */\n")
diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py
--- a/pypy/module/cpyext/cdatetime.py
+++ b/pypy/module/cpyext/cdatetime.py
@@ -2,9 +2,10 @@
from rpython.rtyper.annlowlevel import llhelper
from rpython.rlib.rarithmetic import widen
from pypy.module.cpyext.pyobject import (PyObject, make_ref, make_typedescr,
- decref)
+ decref, as_pyobj, incref)
from pypy.module.cpyext.api import (cpython_api, CANNOT_FAIL, cpython_struct,
- PyObjectFields, cts, parse_dir, bootstrap_function, slot_function)
+ PyObjectFields, cts, parse_dir, bootstrap_function, slot_function,
+ Py_TPFLAGS_HEAPTYPE)
from pypy.module.cpyext.import_ import PyImport_Import
from pypy.module.cpyext.typeobject import PyTypeObjectPtr
from pypy.interpreter.error import OperationError
@@ -31,6 +32,10 @@
w_type = space.getattr(w_datetime, space.newtext("date"))
datetimeAPI.c_DateType = rffi.cast(
PyTypeObjectPtr, make_ref(space, w_type))
+ # convenient place to modify this, needed since the make_typedescr attach
+ # links the "wrong" struct to W_DateTime_Date, which in turn is needed
+ # because datetime, with a tzinfo entry, inherits from date, without one
+ datetimeAPI.c_DateType.c_tp_basicsize = rffi.sizeof(PyObject.TO)
w_type = space.getattr(w_datetime, space.newtext("datetime"))
datetimeAPI.c_DateTimeType = rffi.cast(
@@ -128,6 +133,7 @@
# W_DateTime_Date->tp_dealloc
make_typedescr(W_DateTime_Date.typedef,
basestruct=PyDateTime_DateTime.TO,
+ attach=type_attach,
dealloc=date_dealloc,
)
@@ -138,8 +144,10 @@
def type_attach(space, py_obj, w_obj, w_userdata=None):
'''Fills a newly allocated py_obj from the w_obj
- Can be called with a datetime, or a time
'''
+ if space.type(w_obj).name == 'date':
+ # No tzinfo
+ return
py_datetime = rffi.cast(PyDateTime_Time, py_obj)
w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo'))
if space.is_none(w_tzinfo):
diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py
--- a/pypy/module/cpyext/eval.py
+++ b/pypy/module/cpyext/eval.py
@@ -5,7 +5,7 @@
from rpython.rlib.rarithmetic import widen
from pypy.module.cpyext.api import (
cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, fread, feof, Py_ssize_tP,
- cpython_struct, is_valid_fp)
+ cpython_struct)
from pypy.module.cpyext.pyobject import PyObject
from pypy.module.cpyext.pyerrors import PyErr_SetFromErrno
from pypy.module.cpyext.funcobject import PyCodeObject
@@ -155,22 +155,19 @@
BUF_SIZE = 8192
source = ""
filename = rffi.charp2str(filename)
- buf = lltype.malloc(rffi.CCHARP.TO, BUF_SIZE, flavor='raw')
- if not is_valid_fp(fp):
- lltype.free(buf, flavor='raw')
- PyErr_SetFromErrno(space, space.w_IOError)
- return None
- try:
+ with rffi.scoped_alloc_buffer(BUF_SIZE) as buf:
while True:
- count = fread(buf, 1, BUF_SIZE, fp)
+ try:
+ count = fread(buf.raw, 1, BUF_SIZE, fp)
+ except OSError:
+ PyErr_SetFromErrno(space, space.w_IOError)
+ return
count = rffi.cast(lltype.Signed, count)
- source += rffi.charpsize2str(buf, count)
+ source += rffi.charpsize2str(buf.raw, count)
if count < BUF_SIZE:
if feof(fp):
break
PyErr_SetFromErrno(space, space.w_IOError)
- finally:
- lltype.free(buf, flavor='raw')
return run_string(space, source, filename, start, w_globals, w_locals)
# Undocumented function!
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -6,7 +6,7 @@
from rpython.rlib.rarithmetic import widen
from pypy.module.cpyext.api import (
slot_function, generic_cpy_call, PyObject, Py_ssize_t,
- Py_TPFLAGS_CHECKTYPES, Py_buffer, Py_bufferP, PyTypeObjectPtr, cts)
+ Py_TPFLAGS_CHECKTYPES, Py_buffer, Py_bufferP, PyTypeObjectPtr)
from pypy.module.cpyext.typeobjectdefs import (
unaryfunc, ternaryfunc, binaryfunc,
getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
@@ -143,7 +143,6 @@
def wrap_inquirypred(space, w_self, w_args, func):
func_inquiry = rffi.cast(inquiry, func)
check_num_args(space, w_args, 0)
- args_w = space.fixedview(w_args)
res = generic_cpy_call(space, func_inquiry, w_self)
res = rffi.cast(lltype.Signed, res)
if res == -1:
@@ -423,285 +422,331 @@
return space.newint(generic_cpy_call(space, func_target, w_self, w_other))
-from rpython.rlib.nonconst import NonConstant
+SLOT_FACTORIES = {}
+def slot_factory(tp_name):
+ def decorate(func):
+ SLOT_FACTORIES[tp_name] = func
+ return func
+ return decorate
-def build_slot_tp_function(space, typedef, name):
+
+SLOTS = {}
+ at specialize.memo()
+def get_slot_tp_function(space, typedef, name, method_name):
+ """Return a description of the slot C function to use for the built-in
+ type for 'typedef'. The 'name' is the slot name. This is a memo
+ function that, after translation, returns one of a built-in finite set.
+ """
+ key = (typedef, name)
+ try:
+ return SLOTS[key]
+ except KeyError:
+ slot_func = SLOT_FACTORIES[name](space, typedef, name, method_name)
+ api_func = slot_func.api_func if slot_func else None
+ SLOTS[key] = api_func
+ return api_func
+
+
+def make_unary_slot(space, typedef, name, attr):
w_type = space.gettypeobject(typedef)
-
- handled = False
- # unary functions
- for tp_name, attr in [('tp_as_number.c_nb_int', '__int__'),
- ('tp_as_number.c_nb_long', '__long__'),
- ('tp_as_number.c_nb_float', '__float__'),
- ('tp_as_number.c_nb_negative', '__neg__'),
- ('tp_as_number.c_nb_positive', '__pos__'),
- ('tp_as_number.c_nb_absolute', '__abs__'),
- ('tp_as_number.c_nb_invert', '__invert__'),
- ('tp_as_number.c_nb_index', '__index__'),
- ('tp_as_number.c_nb_hex', '__hex__'),
- ('tp_as_number.c_nb_oct', '__oct__'),
- ('tp_str', '__str__'),
- ('tp_repr', '__repr__'),
- ('tp_iter', '__iter__'),
- ]:
- if name == tp_name:
- slot_fn = w_type.lookup(attr)
- if slot_fn is None:
- return
-
- @slot_function([PyObject], PyObject)
- @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
- def slot_func(space, w_self):
- return space.call_function(slot_fn, w_self)
- handled = True
-
- for tp_name, attr in [('tp_hash', '__hash__'),
- ('tp_as_sequence.c_sq_length', '__len__'),
- ('tp_as_mapping.c_mp_length', '__len__'),
- ]:
- if name == tp_name:
- slot_fn = w_type.lookup(attr)
- if slot_fn is None:
- return
- @slot_function([PyObject], lltype.Signed, error=-1)
- @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
- def slot_func(space, w_obj):
- return space.int_w(space.call_function(slot_fn, w_obj))
- handled = True
-
-
- # binary functions
- for tp_name, attr in [('tp_as_number.c_nb_add', '__add__'),
- ('tp_as_number.c_nb_subtract', '__sub__'),
- ('tp_as_number.c_nb_multiply', '__mul__'),
- ('tp_as_number.c_nb_divide', '__div__'),
- ('tp_as_number.c_nb_remainder', '__mod__'),
- ('tp_as_number.c_nb_divmod', '__divmod__'),
- ('tp_as_number.c_nb_lshift', '__lshift__'),
- ('tp_as_number.c_nb_rshift', '__rshift__'),
- ('tp_as_number.c_nb_and', '__and__'),
- ('tp_as_number.c_nb_xor', '__xor__'),
- ('tp_as_number.c_nb_or', '__or__'),
- ('tp_as_sequence.c_sq_concat', '__add__'),
- ('tp_as_sequence.c_sq_inplace_concat', '__iadd__'),
- ('tp_as_mapping.c_mp_subscript', '__getitem__'),
- ]:
- if name == tp_name:
- slot_fn = w_type.lookup(attr)
- if slot_fn is None:
- return
-
- @slot_function([PyObject, PyObject], PyObject)
- @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
- def slot_func(space, w_self, w_arg):
- return space.call_function(slot_fn, w_self, w_arg)
- handled = True
-
- # binary-with-Py_ssize_t-type
- for tp_name, attr in [('tp_as_sequence.c_sq_item', '__getitem__'),
- ('tp_as_sequence.c_sq_repeat', '__mul__'),
- ('tp_as_sequence.c_sq_repeat', '__mul__'),
- ('tp_as_sequence.c_sq_inplace_repeat', '__imul__'),
- ]:
- if name == tp_name:
- slot_fn = w_type.lookup(attr)
- if slot_fn is None:
- return
-
- @slot_function([PyObject, Py_ssize_t], PyObject)
- @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
- def slot_func(space, w_self, arg):
- return space.call_function(slot_fn, w_self, space.newint(arg))
- handled = True
-
- # ternary functions
- for tp_name, attr in [('tp_as_number.c_nb_power', '__pow__'),
- ]:
- if name == tp_name:
- slot_fn = w_type.lookup(attr)
- if slot_fn is None:
- return
-
- @slot_function([PyObject, PyObject, PyObject], PyObject)
- @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
- def slot_func(space, w_self, w_arg1, w_arg2):
- return space.call_function(slot_fn, w_self, w_arg1, w_arg2)
- handled = True
- # ternary-with-void returning-Py_size_t-type
- for tp_name, attr in [('tp_as_mapping.c_mp_ass_subscript', '__setitem__'),
- ]:
- if name == tp_name:
- slot_ass = w_type.lookup(attr)
- if slot_ass is None:
- return
- slot_del = w_type.lookup('__delitem__')
- if slot_del is None:
- return
-
- @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
- @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
- def slot_func(space, w_self, w_arg1, arg2):
- if arg2:
- w_arg2 = from_ref(space, rffi.cast(PyObject, arg2))
- space.call_function(slot_ass, w_self, w_arg1, w_arg2)
- else:
- space.call_function(slot_del, w_self, w_arg1)
- return 0
- handled = True
- # ternary-Py_size_t-void returning-Py_size_t-type
- for tp_name, attr in [('tp_as_sequence.c_sq_ass_item', '__setitem__'),
- ]:
- if name == tp_name:
- slot_ass = w_type.lookup(attr)
- if slot_ass is None:
- return
- slot_del = w_type.lookup('__delitem__')
- if slot_del is None:
- return
-
- @slot_function([PyObject, lltype.Signed, PyObject], rffi.INT_real, error=-1)
- @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
- def slot_func(space, w_self, arg1, arg2):
- if arg2:
- w_arg2 = from_ref(space, rffi.cast(PyObject, arg2))
- space.call_function(slot_ass, w_self, space.newint(arg1), w_arg2)
- else:
- space.call_function(slot_del, w_self, space.newint(arg1))
- return 0
- handled = True
- if handled:
- pass
- elif name == 'tp_setattro':
- setattr_fn = w_type.lookup('__setattr__')
- delattr_fn = w_type.lookup('__delattr__')
- if setattr_fn is None:
- return
-
- @slot_function([PyObject, PyObject, PyObject], rffi.INT_real,
- error=-1)
- @func_renamer("cpyext_tp_setattro_%s" % (typedef.name,))
- def slot_tp_setattro(space, w_self, w_name, w_value):
- if w_value is not None:
- space.call_function(setattr_fn, w_self, w_name, w_value)
- else:
- space.call_function(delattr_fn, w_self, w_name)
- return 0
- slot_func = slot_tp_setattro
- elif name == 'tp_getattro':
- getattr_fn = w_type.lookup('__getattribute__')
- if getattr_fn is None:
- return
-
- @slot_function([PyObject, PyObject], PyObject)
- @func_renamer("cpyext_tp_getattro_%s" % (typedef.name,))
- def slot_tp_getattro(space, w_self, w_name):
- return space.call_function(getattr_fn, w_self, w_name)
- slot_func = slot_tp_getattro
-
- elif name == 'tp_call':
- call_fn = w_type.lookup('__call__')
- if call_fn is None:
- return
-
- @slot_function([PyObject, PyObject, PyObject], PyObject)
- @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
- def slot_tp_call(space, w_self, w_args, w_kwds):
- args = Arguments(space, [w_self],
- w_stararg=w_args, w_starstararg=w_kwds)
- return space.call_args(call_fn, args)
- slot_func = slot_tp_call
-
- elif name == 'tp_iternext':
- iternext_fn = w_type.lookup('next')
- if iternext_fn is None:
- return
-
- @slot_function([PyObject], PyObject)
- @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
- def slot_tp_iternext(space, w_self):
- try:
- return space.call_function(iternext_fn, w_self)
- except OperationError as e:
- if not e.match(space, space.w_StopIteration):
- raise
- return None
- slot_func = slot_tp_iternext
-
- elif name == 'tp_init':
- init_fn = w_type.lookup('__init__')
- if init_fn is None:
- return
-
- @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
- @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
- def slot_tp_init(space, w_self, w_args, w_kwds):
- args = Arguments(space, [w_self],
- w_stararg=w_args, w_starstararg=w_kwds)
- space.call_args(init_fn, args)
- return 0
- slot_func = slot_tp_init
- elif name == 'tp_new':
- new_fn = w_type.lookup('__new__')
- if new_fn is None:
- return
-
- @slot_function([PyTypeObjectPtr, PyObject, PyObject], PyObject)
- @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
- def slot_tp_new(space, w_self, w_args, w_kwds):
- args = Arguments(space, [w_self],
- w_stararg=w_args, w_starstararg=w_kwds)
- return space.call_args(space.get(new_fn, w_self), args)
- slot_func = slot_tp_new
- elif name == 'tp_as_buffer.c_bf_getbuffer':
- buff_fn = w_type.lookup('__buffer__')
- if buff_fn is not None:
- buff_w = slot_from___buffer__(space, typedef, buff_fn)
- elif typedef.buffer:
- buff_w = slot_from_buffer_w(space, typedef, buff_fn)
- else:
- return
- slot_func = buff_w
- elif name == 'tp_descr_get':
- get_fn = w_type.lookup('__get__')
- if get_fn is None:
- return
-
- @slot_function([PyObject, PyObject, PyObject], PyObject)
- @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
- def slot_tp_descr_get(space, w_self, w_obj, w_value):
- if w_obj is None:
- w_obj = space.w_None
- return space.call_function(get_fn, w_self, w_obj, w_value)
- slot_func = slot_tp_descr_get
- elif name == 'tp_descr_set':
- set_fn = w_type.lookup('__set__')
- delete_fn = w_type.lookup('__delete__')
- if set_fn is None and delete_fn is None:
- return
-
- @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
- @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
- def slot_tp_descr_set(space, w_self, w_obj, w_value):
- if w_value is not None:
- if set_fn is None:
- raise oefmt(space.w_TypeError,
- "%s object has no __set__", typedef.name)
- space.call_function(set_fn, w_self, w_obj, w_value)
- else:
- if delete_fn is None:
- raise oefmt(space.w_TypeError,
- "%s object has no __delete__", typedef.name)
- space.call_function(delete_fn, w_self, w_obj)
- return 0
- slot_func = slot_tp_descr_set
- else:
- # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce
- # tp_as_sequence.c_sq_contains, tp_as_sequence.c_sq_length
- # richcmpfunc(s)
+ slot_fn = w_type.lookup(attr)
+ if slot_fn is None:
return
+ @slot_function([PyObject], PyObject)
+ @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+ def slot_func(space, w_self):
+ return space.call_function(slot_fn, w_self)
return slot_func
+UNARY_SLOTS = [
+ 'tp_as_number.c_nb_int',
+ 'tp_as_number.c_nb_long',
+ 'tp_as_number.c_nb_float',
+ 'tp_as_number.c_nb_negative',
+ 'tp_as_number.c_nb_positive',
+ 'tp_as_number.c_nb_absolute',
+ 'tp_as_number.c_nb_invert',
+ 'tp_as_number.c_nb_index',
+ 'tp_as_number.c_nb_hex',
+ 'tp_as_number.c_nb_oct',
+ 'tp_str',
+ 'tp_repr',
+ 'tp_iter']
+for name in UNARY_SLOTS:
+ slot_factory(name)(make_unary_slot)
+
+def make_unary_slot_int(space, typedef, name, attr):
+ w_type = space.gettypeobject(typedef)
+ slot_fn = w_type.lookup(attr)
+ if slot_fn is None:
+ return
+ @slot_function([PyObject], lltype.Signed, error=-1)
+ @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+ def slot_func(space, w_obj):
+ return space.int_w(space.call_function(slot_fn, w_obj))
+ return slot_func
+
+UNARY_SLOTS_INT = [
+ 'tp_hash',
+ 'tp_as_sequence.c_sq_length',
+ 'tp_as_mapping.c_mp_length',]
+for name in UNARY_SLOTS_INT:
+ slot_factory(name)(make_unary_slot_int)
+
+
+def make_binary_slot(space, typedef, name, attr):
+ w_type = space.gettypeobject(typedef)
+ slot_fn = w_type.lookup(attr)
+ if slot_fn is None:
+ return
+
+ @slot_function([PyObject, PyObject], PyObject)
+ @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+ def slot_func(space, w_self, w_arg):
+ return space.call_function(slot_fn, w_self, w_arg)
+ return slot_func
+
+BINARY_SLOTS = [
+ 'tp_as_number.c_nb_add',
+ 'tp_as_number.c_nb_subtract',
+ 'tp_as_number.c_nb_multiply',
+ 'tp_as_number.c_nb_divide',
+ 'tp_as_number.c_nb_remainder',
+ 'tp_as_number.c_nb_divmod',
+ 'tp_as_number.c_nb_lshift',
+ 'tp_as_number.c_nb_rshift',
+ 'tp_as_number.c_nb_and',
+ 'tp_as_number.c_nb_xor',
+ 'tp_as_number.c_nb_or',
+ 'tp_as_sequence.c_sq_concat',
+ 'tp_as_sequence.c_sq_inplace_concat',
+ 'tp_as_mapping.c_mp_subscript',]
+for name in BINARY_SLOTS:
+ slot_factory(name)(make_binary_slot)
+
+
+def make_binary_slot_int(space, typedef, name, attr):
+ w_type = space.gettypeobject(typedef)
+ slot_fn = w_type.lookup(attr)
+ if slot_fn is None:
+ return
+
+ @slot_function([PyObject, Py_ssize_t], PyObject)
+ @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+ def slot_func(space, w_self, arg):
+ return space.call_function(slot_fn, w_self, space.newint(arg))
+ return slot_func
+
+BINARY_SLOTS_INT = [
+ 'tp_as_sequence.c_sq_item',
+ 'tp_as_sequence.c_sq_repeat',
+ 'tp_as_sequence.c_sq_repeat',
+ 'tp_as_sequence.c_sq_inplace_repeat',]
+for name in BINARY_SLOTS_INT:
+ slot_factory(name)(make_binary_slot_int)
+
+ at slot_factory('tp_as_number.c_nb_power')
+def make_nb_power(space, typedef, name, attr):
+ w_type = space.gettypeobject(typedef)
+ slot_fn = w_type.lookup(attr)
+ if slot_fn is None:
+ return
+
+ @slot_function([PyObject, PyObject, PyObject], PyObject)
+ @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+ def slot_func(space, w_self, w_arg1, w_arg2):
+ return space.call_function(slot_fn, w_self, w_arg1, w_arg2)
+ return slot_func
+
+ at slot_factory('tp_as_mapping.c_mp_ass_subscript')
+def make_sq_set_item(space, typedef, name, attr):
+ w_type = space.gettypeobject(typedef)
+ slot_ass = w_type.lookup(attr)
+ if slot_ass is None:
+ return
+ slot_del = w_type.lookup('__delitem__')
+ if slot_del is None:
+ return
+
+ @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
+ @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+ def slot_func(space, w_self, w_arg1, arg2):
+ if arg2:
+ w_arg2 = from_ref(space, rffi.cast(PyObject, arg2))
+ space.call_function(slot_ass, w_self, w_arg1, w_arg2)
+ else:
+ space.call_function(slot_del, w_self, w_arg1)
+ return 0
+ return slot_func
+
+
+ at slot_factory('tp_as_sequence.c_sq_ass_item')
+def make_sq_ass_item(space, typedef, name, attr):
+ w_type = space.gettypeobject(typedef)
+ slot_ass = w_type.lookup(attr)
+ if slot_ass is None:
+ return
+ slot_del = w_type.lookup('__delitem__')
+ if slot_del is None:
+ return
+
+ @slot_function([PyObject, lltype.Signed, PyObject], rffi.INT_real, error=-1)
+ @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+ def slot_func(space, w_self, arg1, arg2):
+ if arg2:
+ w_arg2 = from_ref(space, rffi.cast(PyObject, arg2))
+ space.call_function(slot_ass, w_self, space.newint(arg1), w_arg2)
+ else:
+ space.call_function(slot_del, w_self, space.newint(arg1))
+ return 0
+ return slot_func
+
+def make_tp_setattro(space, typedef, name, attr):
+ w_type = space.gettypeobject(typedef)
+ setattr_fn = w_type.lookup('__setattr__')
+ delattr_fn = w_type.lookup('__delattr__')
+ if setattr_fn is None or delattr_fn is None:
+ return
+
+ @slot_function([PyObject, PyObject, PyObject], rffi.INT_real,
+ error=-1)
+ @func_renamer("cpyext_tp_setattro_%s" % (typedef.name,))
+ def slot_tp_setattro(space, w_self, w_name, w_value):
+ if w_value is not None:
+ space.call_function(setattr_fn, w_self, w_name, w_value)
+ else:
+ space.call_function(delattr_fn, w_self, w_name)
+ return 0
+ return slot_tp_setattro
+
+ at slot_factory('tp_getattro')
+def make_tp_getattro(space, typedef, name, attr):
+ w_type = space.gettypeobject(typedef)
+ getattr_fn = w_type.lookup('__getattribute__')
+ if getattr_fn is None:
+ return
+
+ @slot_function([PyObject, PyObject], PyObject)
+ @func_renamer("cpyext_tp_getattro_%s" % (typedef.name,))
+ def slot_tp_getattro(space, w_self, w_name):
+ return space.call_function(getattr_fn, w_self, w_name)
+ return slot_tp_getattro
+
+ at slot_factory('tp_call')
+def make_tp_call(space, typedef, name, attr):
+ w_type = space.gettypeobject(typedef)
+ call_fn = w_type.lookup('__call__')
+ if call_fn is None:
+ return
+
+ @slot_function([PyObject, PyObject, PyObject], PyObject)
+ @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+ def slot_tp_call(space, w_self, w_args, w_kwds):
+ args = Arguments(space, [w_self],
+ w_stararg=w_args, w_starstararg=w_kwds)
+ return space.call_args(call_fn, args)
+ return slot_tp_call
+
+ at slot_factory('tp_iternext')
+def make_tp_iternext(space, typedef, name, attr):
+ w_type = space.gettypeobject(typedef)
+ iternext_fn = w_type.lookup('next')
+ if iternext_fn is None:
+ return
+
+ @slot_function([PyObject], PyObject)
+ @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+ def slot_tp_iternext(space, w_self):
+ try:
+ return space.call_function(iternext_fn, w_self)
+ except OperationError as e:
+ if not e.match(space, space.w_StopIteration):
+ raise
+ return None
+ return slot_tp_iternext
+
+ at slot_factory('tp_init')
+def make_tp_init(space, typedef, name, attr):
+ w_type = space.gettypeobject(typedef)
+ init_fn = w_type.lookup('__init__')
+ if init_fn is None:
+ return
+
+ @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
+ @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+ def slot_tp_init(space, w_self, w_args, w_kwds):
+ args = Arguments(space, [w_self],
+ w_stararg=w_args, w_starstararg=w_kwds)
+ space.call_args(init_fn, args)
+ return 0
+ return slot_tp_init
+
+ at slot_factory('tp_new')
+def make_tp_new(space, typedef, name, attr):
+ w_type = space.gettypeobject(typedef)
+ new_fn = w_type.lookup('__new__')
+ if new_fn is None:
+ return
+
+ @slot_function([PyTypeObjectPtr, PyObject, PyObject], PyObject)
+ @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+ def slot_tp_new(space, w_self, w_args, w_kwds):
+ args = Arguments(space, [w_self],
+ w_stararg=w_args, w_starstararg=w_kwds)
+ return space.call_args(space.get(new_fn, w_self), args)
+ return slot_tp_new
+
+ at slot_factory('tp_as_buffer.c_bf_getbuffer')
+def make_bf_getbuffer(space, typedef, name, attr):
+ w_type = space.gettypeobject(typedef)
+ buff_fn = w_type.lookup('__buffer__')
+ if buff_fn is not None:
+ return slot_from___buffer__(space, typedef, buff_fn)
+ elif typedef.buffer:
+ return slot_from_buffer_w(space, typedef)
+ else:
+ return None
+
+ at slot_factory('tp_descr_get')
+def make_tp_descr_get(space, typedef, name, attr):
+ w_type = space.gettypeobject(typedef)
+ get_fn = w_type.lookup('__get__')
+ if get_fn is None:
+ return
+
+ @slot_function([PyObject, PyObject, PyObject], PyObject)
+ @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+ def slot_tp_descr_get(space, w_self, w_obj, w_value):
+ if w_obj is None:
+ w_obj = space.w_None
+ return space.call_function(get_fn, w_self, w_obj, w_value)
+ return slot_tp_descr_get
+
+ at slot_factory('tp_descr_set')
+def make_tp_descr_set(space, typedef, name, attr):
+ w_type = space.gettypeobject(typedef)
+ set_fn = w_type.lookup('__set__')
+ delete_fn = w_type.lookup('__delete__')
+ if set_fn is None and delete_fn is None:
+ return
+
+ @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
+ @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+ def slot_tp_descr_set(space, w_self, w_obj, w_value):
+ if w_value is not None:
+ if set_fn is None:
+ raise oefmt(space.w_TypeError,
+ "%s object has no __set__", typedef.name)
+ space.call_function(set_fn, w_self, w_obj, w_value)
+ else:
+ if delete_fn is None:
+ raise oefmt(space.w_TypeError,
+ "%s object has no __delete__", typedef.name)
+ space.call_function(delete_fn, w_self, w_obj)
+ return 0
+ return slot_tp_descr_set
+
def slot_from___buffer__(space, typedef, buff_fn):
name = 'bf_getbuffer'
@@ -730,7 +775,7 @@
return 0
return buff_w
-def slot_from_buffer_w(space, typedef, buff_fn):
+def slot_from_buffer_w(space, typedef):
name = 'bf_getbuffer'
@slot_function([PyObject, Py_bufferP, rffi.INT_real],
rffi.INT_real, error=-1)
@@ -765,6 +810,28 @@
missing_wrapper.__name__ = name
globals()[name] = missing_wrapper
+def make_missing_slot(space, typedef, name, attr):
+ return None
+
+missing_builtin_slots = [
+ 'tp_print', 'tp_compare', 'tp_getattr', 'tp_setattr', 'tp_setattro',
+ 'tp_richcompare', 'tp_del', 'tp_as_buffer.c_bf_getwritebuffer',
+ 'tp_as_number.c_nb_nonzero', 'tp_as_number.c_nb_coerce',
+ 'tp_as_number.c_nb_inplace_add', 'tp_as_number.c_nb_inplace_subtract',
+ 'tp_as_number.c_nb_inplace_multiply', 'tp_as_number.c_nb_inplace_divide',
+ 'tp_as_number.c_nb_inplace_remainder', 'tp_as_number.c_nb_inplace_power',
+ 'tp_as_number.c_nb_inplace_lshift', 'tp_as_number.c_nb_inplace_rshift',
+ 'tp_as_number.c_nb_inplace_and', 'tp_as_number.c_nb_inplace_xor',
+ 'tp_as_number.c_nb_inplace_or',
+ 'tp_as_number.c_nb_floor_divide', 'tp_as_number.c_nb_true_divide',
+ 'tp_as_number.c_nb_inplace_floor_divide', 'tp_as_number.c_nb_inplace_true_divide',
+ 'tp_as_sequence.c_sq_slice', 'tp_as_sequence.c_sq_ass_slice',
+ 'tp_as_sequence.c_sq_contains',
+ 'tp_as_buffer.c_bf_getreadbuffer',
+ ]
+for name in missing_builtin_slots:
+ slot_factory(name)(make_missing_slot)
+
PyWrapperFlag_KEYWORDS = 1
diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py
--- a/pypy/module/cpyext/test/test_datetime.py
+++ b/pypy/module/cpyext/test/test_datetime.py
@@ -82,16 +82,6 @@
date = datetime.datetime.fromtimestamp(0)
assert space.unwrap(space.str(w_date)) == str(date)
- def test_tzinfo(self, space):
- w_tzinfo = space.appexec(
- [], """():
- from datetime import tzinfo
- return tzinfo()
- """)
- assert PyTZInfo_Check(space, w_tzinfo)
- assert PyTZInfo_CheckExact(space, w_tzinfo)
- assert not PyTZInfo_Check(space, space.w_None)
-
class AppTestDatetime(AppTestCpythonExtensionBase):
def test_CAPI(self):
module = self.import_extension('foo', [
@@ -272,3 +262,81 @@
2000, 6, 6, 6, 6, 6, 6)
assert module.test_time_macros() == datetime.time(6, 6, 6, 6)
assert module.test_delta_macros() == datetime.timedelta(6, 6, 6)
+
+ def test_tzinfo(self):
+ module = self.import_extension('foo', [
+ ("time_with_tzinfo", "METH_O",
+ """ PyDateTime_IMPORT;
+ return PyDateTimeAPI->Time_FromTime(
+ 6, 6, 6, 6, args, PyDateTimeAPI->TimeType);
+ """),
+ ("datetime_with_tzinfo", "METH_O",
+ """
+ PyObject * obj;
+ int tzrefcnt = args->ob_refcnt;
+ PyDateTime_IMPORT;
+ obj = PyDateTimeAPI->DateTime_FromDateAndTime(
+ 2000, 6, 6, 6, 6, 6, 6, args,
+ PyDateTimeAPI->DateTimeType);
+ if (!((PyDateTime_DateTime*)obj)->hastzinfo)
+ {
+ Py_DECREF(obj);
+ PyErr_SetString(PyExc_ValueError, "missing tzinfo");
+ return NULL;
+ }
+ if (((PyDateTime_DateTime*)obj)->tzinfo->ob_refcnt <= tzrefcnt)
+ {
+ Py_DECREF(obj);
+ PyErr_SetString(PyExc_ValueError, "tzinfo refcnt not incremented");
+ return NULL;
+ }
+ return obj;
+
+ """),
+ ], prologue='#include "datetime.h"\n')
+ from datetime import tzinfo, datetime, timedelta, time
+ # copied from datetime documentation
+ class GMT1(tzinfo):
+ def utcoffset(self, dt):
+ return timedelta(hours=1) + self.dst(dt)
+ def dst(self, dt):
+ return timedelta(0)
+ def tzname(self,dt):
+ return "GMT +1"
+ gmt1 = GMT1()
+ dt1 = module.time_with_tzinfo(gmt1)
+ assert dt1 == time(6, 6, 6, 6, gmt1)
+ assert '+01' in str(dt1)
+ assert module.datetime_with_tzinfo(gmt1) == datetime(
+ 2000, 6, 6, 6, 6, 6, 6, gmt1)
+
+ def test_checks(self):
+ module = self.import_extension('foo', [
+ ("checks", "METH_O",
+ """ PyDateTime_IMPORT;
+ return PyTuple_Pack(10,
+ PyBool_FromLong(PyDateTime_Check(args)),
+ PyBool_FromLong(PyDateTime_CheckExact(args)),
+ PyBool_FromLong(PyDate_Check(args)),
+ PyBool_FromLong(PyDate_CheckExact(args)),
+ PyBool_FromLong(PyTime_Check(args)),
+ PyBool_FromLong(PyTime_CheckExact(args)),
+ PyBool_FromLong(PyDelta_Check(args)),
+ PyBool_FromLong(PyDelta_CheckExact(args)),
+ PyBool_FromLong(PyTZInfo_Check(args)),
+ PyBool_FromLong(PyTZInfo_CheckExact(args))
+ );
+ """),
+ ], prologue='#include "datetime.h"\n')
+ from datetime import tzinfo, datetime, timedelta, time, date
+ o = date(1, 1, 1)
+ assert module.checks(o) == (False,) * 2 + (True,) * 2 + (False,) * 6
+ o = time(1, 1, 1)
+ assert module.checks(o) == (False,) * 4 + (True,) * 2 + (False,) * 4
+ o = timedelta(1, 1, 1)
+ assert module.checks(o) == (False,) * 6 + (True,) * 2 + (False,) * 2
+ o = datetime(1, 1, 1)
+ assert module.checks(o) == (True,) * 3 + (False,) * 7 # isinstance(datetime, date)
+ o = tzinfo()
+ assert module.checks(o) == (False,) * 8 + (True,) * 2
+
diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -13,7 +13,7 @@
PyEval_GetBuiltins, PyEval_GetLocals, PyEval_GetGlobals,
_PyEval_SliceIndex)
from pypy.module.cpyext.api import (
- c_fopen, c_fclose, c_fileno, Py_ssize_tP, is_valid_fd)
+ c_fopen, c_fclose, c_fileno, Py_ssize_tP)
from pypy.module.cpyext.pyobject import get_w_obj_and_decref
from pypy.interpreter.gateway import interp2app
from pypy.interpreter.error import OperationError
@@ -150,7 +150,6 @@
os.close(c_fileno(fp))
with raises_w(space, IOError):
PyRun_File(space, fp, filename, Py_file_input, w_globals, w_locals)
- if is_valid_fd(c_fileno(fp)):
c_fclose(fp)
rffi.free_charp(filename)
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -31,7 +31,7 @@
PyObject, make_ref, from_ref, get_typedescr, make_typedescr,
track_reference, decref, as_pyobj)
from pypy.module.cpyext.slotdefs import (
- slotdefs_for_tp_slots, slotdefs_for_wrappers, build_slot_tp_function,
+ slotdefs_for_tp_slots, slotdefs_for_wrappers, get_slot_tp_function,
llslot)
from pypy.module.cpyext.state import State
from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne
@@ -226,22 +226,6 @@
dict_w[name] = w_descr
i += 1
-SLOTS = {}
- at specialize.memo()
-def get_slot_tp_function(space, typedef, name):
- """Return a description of the slot C function to use for the built-in
- type for 'typedef'. The 'name' is the slot name. This is a memo
- function that, after translation, returns one of a built-in finite set.
- """
- key = (typedef, name)
- try:
- return SLOTS[key]
- except KeyError:
- slot_func = build_slot_tp_function(space, typedef, name)
- api_func = slot_func.api_func if slot_func else None
- SLOTS[key] = api_func
- return api_func
-
missing_slots={}
def warn_missing_slot(space, method_name, slot_name, w_type):
if not we_are_translated():
@@ -281,12 +265,12 @@
def update_all_slots_builtin(space, w_type, pto):
typedef = w_type.layout.typedef
for method_name, slot_name, slot_names, slot_apifunc in slotdefs_for_tp_slots:
- slot_apifunc = get_slot_tp_function(space, typedef, slot_name)
+ slot_apifunc = get_slot_tp_function(space, typedef, slot_name, method_name)
if not slot_apifunc:
warn_missing_slot(space, method_name, slot_name, w_type)
continue
- slot_func_helper = slot_apifunc.get_llhelper(space)
- fill_slot(space, pto, w_type, slot_names, slot_func_helper)
+ slot_llfunc = slot_apifunc.get_llhelper(space)
+ fill_slot(space, pto, w_type, slot_names, slot_llfunc)
@specialize.arg(3)
def fill_slot(space, pto, w_type, slot_names, slot_func_helper):
@@ -332,7 +316,7 @@
def add_operators(space, dict_w, pto):
from pypy.module.cpyext.object import PyObject_HashNotImplemented
- hash_not_impl = PyObject_HashNotImplemented.api_func.get_llhelper(space)
+ hash_not_impl = llslot(space, PyObject_HashNotImplemented)
for method_name, slot_names, wrapper_func, wrapper_func_kwds, doc in slotdefs_for_wrappers:
if method_name in dict_w:
continue
diff --git a/pypy/module/faulthandler/handler.py b/pypy/module/faulthandler/handler.py
--- a/pypy/module/faulthandler/handler.py
+++ b/pypy/module/faulthandler/handler.py
@@ -1,6 +1,5 @@
import os
from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
-from rpython.rlib.rposix import is_valid_fd
from rpython.rlib.rarithmetic import widen, ovfcheck_float_to_longlong
from rpython.rlib.objectmodel import keepalive_until_here
from rpython.rtyper.annlowlevel import llhelper
@@ -35,7 +34,7 @@
raise oefmt(space.w_RuntimeError, "sys.stderr is None")
elif space.isinstance_w(w_file, space.w_int):
fd = space.c_int_w(w_file)
- if fd < 0 or not is_valid_fd(fd):
+ if fd < 0:
raise oefmt(space.w_ValueError,
"file is not a valid file descriptor")
return fd, None
diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py
--- a/pypy/module/gc/app_referents.py
+++ b/pypy/module/gc/app_referents.py
@@ -72,7 +72,7 @@
def __repr__(self):
if self._s.total_memory_pressure != -1:
- extra = "\nmemory pressure: %s" % self.total_memory_pressure
+ extra = "\n memory pressure: %s" % self.total_memory_pressure
else:
extra = ""
return """Total memory consumed:
@@ -109,5 +109,5 @@
self.memory_allocated_sum)
-def get_stats():
- return GcStats(gc._get_stats())
+def get_stats(memory_pressure=False):
+ return GcStats(gc._get_stats(memory_pressure=memory_pressure))
diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py
--- a/pypy/module/posix/app_posix.py
+++ b/pypy/module/posix/app_posix.py
@@ -1,7 +1,6 @@
# NOT_RPYTHON
from _structseq import structseqtype, structseqfield
-from __pypy__ import validate_fd
# XXX we need a way to access the current module's globals more directly...
import sys
@@ -80,25 +79,30 @@
f_flag = structseqfield(8)
f_namemax = structseqfield(9)
+# Capture file.fdopen at import time, as some code replaces
+# __builtins__.file with a custom function.
+_fdopen = file.fdopen
+
if osname == 'posix':
# POSIX: we want to check the file descriptor when fdopen() is called,
# not later when we read or write data. So we call fstat(), letting
# it raise if fd is invalid.
- _validate_fd = posix.fstat
+ def fdopen(fd, mode='r', buffering=-1):
+ """fdopen(fd [, mode='r' [, buffering]]) -> file_object
+
+ Return an open file object connected to a file descriptor."""
+ try:
+ posix.fstat(fd)
+ except OSError as e:
+ raise IOError(e.errno, e.message)
+ return _fdopen(fd, mode, buffering)
+
else:
- _validate_fd = validate_fd
+ def fdopen(fd, mode='r', buffering=-1):
+ """fdopen(fd [, mode='r' [, buffering]]) -> file_object
-# Capture file.fdopen at import time, as some code replaces
-# __builtins__.file with a custom function.
-_fdopen = file.fdopen
-
-def fdopen(fd, mode='r', buffering=-1):
- """fdopen(fd [, mode='r' [, buffering]]) -> file_object
-
- Return an open file object connected to a file descriptor."""
- _validate_fd(fd)
- return _fdopen(fd, mode, buffering)
-
+ Return an open file object connected to a file descriptor."""
+ return _fdopen(fd, mode, buffering)
def tmpfile():
"""Create a temporary file.
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -18,7 +18,7 @@
USEMODULES += ['fcntl']
else:
# On windows, os.popen uses the subprocess module
- USEMODULES += ['_rawffi', 'thread', 'signal']
+ USEMODULES += ['_rawffi', 'thread', 'signal', '_cffi_backend']
def setup_module(mod):
mod.space = gettestobjspace(usemodules=USEMODULES)
@@ -300,8 +300,13 @@
# There used to be code here to ensure that fcntl is not faked
# but we can't do that cleanly any more
- exc = raises(OSError, posix.fdopen, fd)
- assert exc.value.errno == errno.EBADF
+ try:
+ fid = posix.fdopen(fd)
+ fid.read(10)
+ except IOError as e:
+ assert e.errno == errno.EBADF
+ else:
+ assert False, "using result of fdopen(fd) on closed file must raise"
def test_fdopen_hackedbuiltins(self):
"Same test, with __builtins__.file removed"
@@ -331,8 +336,17 @@
path = self.path
posix = self.posix
fd = posix.open(path, posix.O_RDONLY)
- exc = raises(OSError, posix.fdopen, fd, 'w')
- assert str(exc.value) == "[Errno 22] Invalid argument"
+ # compatability issue - using Visual Studio 10 and above no
+ # longer raises on fid creation, only when _using_ fid
+ # win32 python2 raises IOError on flush(), win32 python3 raises OSError
+ try:
+ fid = posix.fdopen(fd, 'w')
+ fid.write('abc')
+ fid.flush()
+ except (OSError, IOError) as e:
+ assert e.errno in (9, 22)
+ else:
+ assert False, "expected OSError"
posix.close(fd) # fd should not be closed
def test_getcwd(self):
diff --git a/pypy/module/test_lib_pypy/pyrepl/test_functional.py b/pypy/module/test_lib_pypy/pyrepl/test_functional.py
deleted file mode 100644
--- a/pypy/module/test_lib_pypy/pyrepl/test_functional.py
+++ /dev/null
@@ -1,27 +0,0 @@
-# Copyright 2000-2007 Michael Hudson-Doyle <micahel at gmail.com>
-# Maciek Fijalkowski
-# License: MIT
-# some functional tests, to see if this is really working
-
-import pytest
-import sys
-
-
-def pytest_funcarg__child(request):
- try:
- import pexpect
- except ImportError:
- pytest.skip("no pexpect module")
- except SyntaxError:
- pytest.skip('pexpect wont work on py3k')
- child = pexpect.spawn(sys.executable, ['-S'], timeout=10)
- child.logfile = sys.stdout
- child.sendline('from pyrepl.python_reader import main')
- child.sendline('main()')
- return child
-
-
-def test_basic(child):
- child.sendline('a = 3')
- child.sendline('a')
- child.expect('3')
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -151,9 +151,10 @@
if (rffi.TIME_T in args or rffi.TIME_TP in args
or result in (rffi.TIME_T, rffi.TIME_TP)):
name = '_' + name + '64'
+ _calling_conv = kwds.pop('calling_conv', calling_conv)
return rffi.llexternal(name, args, result,
compilation_info=eci,
- calling_conv=calling_conv,
+ calling_conv=_calling_conv,
releasegil=False,
**kwds)
@@ -187,20 +188,34 @@
"RPY_EXTERN "
"int pypy_get_daylight();\n"
"RPY_EXTERN "
- "char** pypy_get_tzname();\n"
+ "int pypy_get_tzname(size_t, int, char*);\n"
"RPY_EXTERN "
"void pypy__tzset();"],
separate_module_sources = ["""
- long pypy_get_timezone() { return timezone; }
- int pypy_get_daylight() { return daylight; }
- char** pypy_get_tzname() { return tzname; }
- void pypy__tzset() { _tzset(); }
+ long pypy_get_timezone() {
+ long timezone;
+ _get_timezone(&timezone);
+ return timezone;
+ };
+ int pypy_get_daylight() {
+ int daylight;
+ _get_daylight(&daylight);
+ return daylight;
+ };
+ int pypy_get_tzname(size_t len, int index, char * tzname) {
+ size_t s;
+ errno_t ret = _get_tzname(&s, tzname, len, index);
+ return (int)s;
+ };
+ void pypy__tzset() { _tzset(); }
"""])
# Ensure sure that we use _tzset() and timezone from the same C Runtime.
c_tzset = external('pypy__tzset', [], lltype.Void, win_eci)
c_get_timezone = external('pypy_get_timezone', [], rffi.LONG, win_eci)
c_get_daylight = external('pypy_get_daylight', [], rffi.INT, win_eci)
- c_get_tzname = external('pypy_get_tzname', [], rffi.CCHARPP, win_eci)
+ c_get_tzname = external('pypy_get_tzname',
+ [rffi.SIZE_T, rffi.INT, rffi.CCHARP],
+ rffi.INT, win_eci, calling_conv='c')
c_strftime = external('strftime', [rffi.CCHARP, rffi.SIZE_T, rffi.CCHARP, TM_P],
rffi.SIZE_T)
@@ -221,8 +236,11 @@
timezone = c_get_timezone()
altzone = timezone - 3600
daylight = c_get_daylight()
- tzname_ptr = c_get_tzname()
- tzname = rffi.charp2str(tzname_ptr[0]), rffi.charp2str(tzname_ptr[1])
+ with rffi.scoped_alloc_buffer(100) as buf:
+ s = c_get_tzname(100, 0, buf.raw)
+ tzname[0] = buf.str(s)
+ s = c_get_tzname(100, 1, buf.raw)
+ tzname[1] = buf.str(s)
if _POSIX:
if _CYGWIN:
diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py
--- a/rpython/rlib/clibffi.py
+++ b/rpython/rlib/clibffi.py
@@ -296,7 +296,8 @@
def get_libc_name():
return rwin32.GetModuleFileName(get_libc_handle())
- assert "msvcr" in get_libc_name().lower(), \
+ libc_name = get_libc_name().lower()
+ assert "msvcr" in libc_name or 'ucrtbase' in libc_name, \
"Suspect msvcrt library: %s" % (get_libc_name(),)
elif _MINGW:
def get_libc_name():
diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -173,10 +173,10 @@
def create_fdopen_rfile(fd, mode="r", buffering=-1):
newmode = _sanitize_mode(mode)
- rposix.validate_fd(fd)
ll_mode = rffi.str2charp(newmode)
try:
- ll_file = c_fdopen(fd, ll_mode)
+ with rposix.FdValidator(fd):
+ ll_file = c_fdopen(fd, ll_mode)
if not ll_file:
errno = rposix.get_saved_errno()
raise OSError(errno, os.strerror(errno))
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -39,11 +39,12 @@
if os.name == 'nt':
if platform.name == 'msvc':
- includes=['errno.h','stdio.h']
+ includes=['errno.h','stdio.h', 'stdlib.h']
else:
includes=['errno.h','stdio.h', 'stdint.h']
separate_module_sources =['''
- /* Lifted completely from CPython 3.3 Modules/posix_module.c */
+ /* Lifted completely from CPython 3 Modules/posixmodule.c */
+ #if defined _MSC_VER && _MSC_VER >= 1400 && _MSC_VER < 1900
#include <malloc.h> /* for _msize */
typedef struct {
intptr_t osfhnd;
@@ -95,6 +96,46 @@
errno = EBADF;
return 0;
}
+ RPY_EXTERN void* enter_suppress_iph(void) {return (void*)NULL;};
+ RPY_EXTERN void exit_suppress_iph(void* handle) {};
+ #elif defined _MSC_VER
+ RPY_EXTERN int _PyVerify_fd(int fd)
+ {
+ return 1;
+ }
+ static void __cdecl _Py_silent_invalid_parameter_handler(
+ wchar_t const* expression,
+ wchar_t const* function,
+ wchar_t const* file,
+ unsigned int line,
+ uintptr_t pReserved) {
+ wprintf(L"Invalid parameter detected in function %s."
+ L" File: %s Line: %d\\n", function, file, line);
+ wprintf(L"Expression: %s\\n", expression);
+ }
+
+ RPY_EXTERN void* enter_suppress_iph(void)
+ {
+ void* ret = _set_thread_local_invalid_parameter_handler(_Py_silent_invalid_parameter_handler);
+ /*fprintf(stdout, "setting %p returning %p\\n", (void*)_Py_silent_invalid_parameter_handler, ret);*/
+ return ret;
+ }
+ RPY_EXTERN void exit_suppress_iph(void* old_handler)
+ {
+ void * ret;
+ _invalid_parameter_handler _handler = (_invalid_parameter_handler)old_handler;
+ ret = _set_thread_local_invalid_parameter_handler(_handler);
+ /*fprintf(stdout, "exiting, setting %p returning %p\\n", old_handler, ret);*/
+ }
+
+ #else
+ RPY_EXTERN int _PyVerify_fd(int fd)
+ {
+ return 1;
+ }
+ RPY_EXTERN void* enter_suppress_iph(void) {return (void*)NULL;};
+ RPY_EXTERN void exit_suppress_iph(void* handle) {};
+ #endif
''',]
post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);']
else:
@@ -193,50 +234,6 @@
else:
rthread.tlfield_rpy_errno.setraw(_get_errno())
# ^^^ keep fork() up-to-date too, below
-
-
-if os.name == 'nt':
- is_valid_fd = jit.dont_look_inside(rffi.llexternal(
- "_PyVerify_fd", [rffi.INT], rffi.INT,
- compilation_info=errno_eci,
- ))
- @enforceargs(int)
- def validate_fd(fd):
- if not is_valid_fd(fd):
- from errno import EBADF
- raise OSError(EBADF, 'Bad file descriptor')
-
- def _bound_for_write(fd, count):
- if count > 32767 and c_isatty(fd):
More information about the pypy-commit
mailing list