[pypy-commit] pypy default: merge msvc14 which allows using Visual Studio 17 compiler suite
mattip
pypy.commits at gmail.com
Mon Feb 12 14:27:06 EST 2018
Author: Matti Picus <matti.picus at gmail.com>
Branch:
Changeset: r93811:496d05d4758e
Date: 2018-02-12 14:06 -0500
http://bitbucket.org/pypy/pypy/changeset/496d05d4758e/
Log: merge msvc14 which allows using Visual Studio 17 compiler suite
diff too long, truncating to 2000 out of 2842 lines
diff --git a/get_externals.py b/get_externals.py
new file mode 100644
--- /dev/null
+++ b/get_externals.py
@@ -0,0 +1,69 @@
+'''Get external dependencies for building PyPy
+they will end up in the platform.host().basepath, something like repo-root/external
+'''
+
+from __future__ import print_function
+
+import argparse
+import os
+import zipfile
+from subprocess import Popen, PIPE
+from rpython.translator.platform import host
+
+def runcmd(cmd, verbose):
+ stdout = stderr = ''
+ report = False
+ try:
+ p = Popen(cmd, stdout=PIPE, stderr=PIPE)
+ stdout, stderr = p.communicate()
+ if p.wait() != 0 or verbose:
+ report = True
+ except Exception as e:
+ stderr = str(e) + '\n' + stderr
+ report = True
+ if report:
+ print('running "%s" returned\n%s\n%s' % (' '.join(cmd), stdout, stderr))
+ if stderr:
+ raise RuntimeError(stderr)
+
+def checkout_repo(dest='externals', org='pypy', branch='default', verbose=False):
+ url = 'https://bitbucket.org/{}/externals'.format(org)
+ if not os.path.exists(dest):
+ cmd = ['hg','clone',url,dest]
+ runcmd(cmd, verbose)
+ cmd = ['hg','-R', dest, 'update',branch]
+ runcmd(cmd, verbose)
+
+def extract_zip(externals_dir, zip_path):
+ with zipfile.ZipFile(os.fspath(zip_path)) as zf:
+ zf.extractall(os.fspath(externals_dir))
+ return externals_dir / zf.namelist()[0].split('/')[0]
+
+def parse_args():
+ p = argparse.ArgumentParser()
+ p.add_argument('-v', '--verbose', action='store_true')
+ p.add_argument('-O', '--organization',
+ help='Organization owning the deps repos', default='pypy')
+ p.add_argument('-e', '--externals', default=host.externals,
+ help='directory in which to store dependencies',
+ )
+ p.add_argument('-b', '--branch', default=host.externals_branch,
+ help='branch to check out',
+ )
+ p.add_argument('-p', '--platform', default=None,
+ help='someday support cross-compilation, ignore for now',
+ )
+ return p.parse_args()
+
+
+def main():
+ args = parse_args()
+ checkout_repo(
+ dest=args.externals,
+ org=args.organization,
+ branch=args.branch,
+ verbose=args.verbose,
+ )
+
+if __name__ == '__main__':
+ main()
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -39,10 +39,24 @@
.. _Microsoft Visual C++ Compiler for Python 2.7: https://www.microsoft.com/en-us/download/details.aspx?id=44266
+Installing "Build Tools for Visual Studio 2017" (for Python 3)
+--------------------------------------------------------------
+
+As documented in the CPython Wiki_, CPython now recommends Visual C++ version
+14.0. A compact version of the compiler suite can be obtained from Microsoft_
+downloads, search the page for "Build Tools for Visual Studio 2017".
+
+You will also need to install the the `Windows SDK`_ in order to use the
+`mt.exe` mainfest compiler.
+
+.. _Wiki: https://wiki.python.org/moin/WindowsCompilers
+.. _Microsoft: https://www.visualstudio.com/downloads
+.. _`Windows SDK`: https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk
+
Translating PyPy with Visual Studio
-----------------------------------
-We routinely test translation using v9, also known as Visual Studio 2008.
+We routinely test translation of PyPy 2.7 using v9 and PyPy 3 with vc14.
Other configurations may work as well.
The translation scripts will set up the appropriate environment variables
@@ -82,8 +96,8 @@
.. _build instructions: http://pypy.org/download.html#building-from-source
-Setting Up Visual Studio for building SSL in Python3
-----------------------------------------------------
+Setting Up Visual Studio 9.0 for building SSL in Python3
+--------------------------------------------------------
On Python3, the ``ssl`` module is based on ``cffi``, and requires a build step after
translation. However ``distutils`` does not support the Micorosft-provided Visual C
@@ -132,243 +146,14 @@
Installing external packages
----------------------------
-On Windows, there is no standard place where to download, build and
-install third-party libraries. We recommend installing them in the parent
-directory of the pypy checkout. For example, if you installed pypy in
-``d:\pypy\trunk\`` (This directory contains a README file), the base
-directory is ``d:\pypy``. You must then set the
-INCLUDE, LIB and PATH (for DLLs) environment variables appropriately.
+We uses a `repository` parallel to pypy to hold binary compiled versions of the
+build dependencies for windows. As part of the `rpython` setup stage, environment
+variables will be set to use these dependencies. The repository has a README
+file on how to replicate, and a branch for each supported platform. You may run
+ the `get_externals.py` utility to checkout the proper branch for your platform
+and PyPy version.
-
-Abridged method (using Visual Studio 2008)
-------------------------------------------
-
-Download the versions of all the external packages from
-https://bitbucket.org/pypy/pypy/downloads/local_59.zip
-(for post-5.8 builds) with sha256 checksum
-``6344230e90ab7a9cb84efbae1ba22051cdeeb40a31823e0808545b705aba8911``
-https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip
-(to reproduce 5.8 builds) with sha256 checksum
-``fbe769bf3a4ab6f5a8b0a05b61930fc7f37da2a9a85a8f609cf5a9bad06e2554`` or
-https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip
-(for 2.4 release and later) or
-https://bitbucket.org/pypy/pypy/downloads/local.zip
-(for pre-2.4 versions)
-Then expand it into the base directory (base_dir) and modify your environment
-to reflect this::
-
- set PATH=<base_dir>\bin;%PATH%
- set INCLUDE=<base_dir>\include;%INCLUDE%
- set LIB=<base_dir>\lib;%LIB%
-
-Now you should be good to go. If you choose this method, you do not need
-to download/build anything else.
-
-Nonabridged method (building from scratch)
-------------------------------------------
-
-If you want to, you can rebuild everything from scratch by continuing.
-
-
-The Boehm garbage collector
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-This library is needed if you plan to use the ``--gc=boehm`` translation
-option (this is the default at some optimization levels like ``-O1``,
-but unneeded for high-performance translations like ``-O2``).
-You may get it at
-http://hboehm.info/gc/gc_source/gc-7.1.tar.gz
-
-Versions 7.0 and 7.1 are known to work; the 6.x series won't work with
-RPython. Unpack this folder in the base directory.
-The default GC_abort(...) function in misc.c will try to open a MessageBox.
-You may want to disable this with the following patch::
-
- --- a/misc.c Sun Apr 20 14:08:27 2014 +0300
- +++ b/misc.c Sun Apr 20 14:08:37 2014 +0300
- @@ -1058,7 +1058,7 @@
- #ifndef PCR
- void GC_abort(const char *msg)
- {
- -# if defined(MSWIN32)
- +# if 0 && defined(MSWIN32)
- (void) MessageBoxA(NULL, msg, "Fatal error in gc", MB_ICONERROR|MB_OK);
- # else
- GC_err_printf("%s\n", msg);
-
-Then open a command prompt::
-
- cd gc-7.1
- nmake -f NT_THREADS_MAKEFILE
- copy Release\gc.dll <somewhere in the PATH>
-
-
-The zlib compression library
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Download http://www.gzip.org/zlib/zlib-1.2.11.tar.gz and extract it in
-the base directory. Then compile::
-
- cd zlib-1.2.11
- nmake -f win32\Makefile.msc
- copy zlib.lib <somewhere in LIB>
- copy zlib.h zconf.h <somewhere in INCLUDE>
- copy zlib1.dll <in PATH> # (needed for tests via ll2ctypes)
-
-
-The bz2 compression library
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Get the same version of bz2 used by python and compile as a static library::
-
- svn export http://svn.python.org/projects/external/bzip2-1.0.6
- cd bzip2-1.0.6
- nmake -f makefile.msc
- copy libbz2.lib <somewhere in LIB>
- copy bzlib.h <somewhere in INCLUDE>
-
-
-The sqlite3 database library
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-PyPy uses cffi to interact with sqlite3.dll. Only the dll is needed, the cffi
-wrapper is compiled when the module is imported for the first time.
-The sqlite3.dll should be version 3.8.11 for CPython2.7 compatablility.
-
-
-The expat XML parser
-~~~~~~~~~~~~~~~~~~~~
-
-CPython compiles expat from source as part of the build. PyPy uses the same
-code base, but expects to link to a static lib of expat. Here are instructions
-to reproduce the static lib in version 2.2.4.
-
-Download the source code of expat: https://github.com/libexpat/libexpat.
-``git checkout`` the proper tag, in this case ``R_2_2_4``. Run
-``vcvars.bat`` to set up the visual compiler tools, and CD into the source
-directory. Create a file ``stdbool.h`` with the content
-
-.. code-block:: c
-
- #pragma once
-
- #define false 0
- #define true 1
-
- #define bool int
-
-and put it in a place on the ``INCLUDE`` path, or create it in the local
-directory and add ``.`` to the ``INCLUDE`` path::
-
- SET INCLUDE=%INCLUDE%;.
-
-Then compile all the ``*.c`` file into ``*.obj``::
-
- cl.exe /nologo /MD /O2 *c /c
- rem for debug
- cl.exe /nologo /MD /O0 /Ob0 /Zi *c /c
-
-You may need to move some variable declarations to the beginning of the
-function, to be compliant with C89 standard. Here is the diff for version 2.2.4
-
-.. code-block:: diff
-
- diff --git a/expat/lib/xmltok.c b/expat/lib/xmltok.c
- index 007aed0..a2dcaad 100644
- --- a/expat/lib/xmltok.c
- +++ b/expat/lib/xmltok.c
- @@ -399,19 +399,21 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc),
- /* Avoid copying partial characters (due to limited space). */
- const ptrdiff_t bytesAvailable = fromLim - *fromP;
- const ptrdiff_t bytesStorable = toLim - *toP;
- + const char * fromLimBefore;
- + ptrdiff_t bytesToCopy;
- if (bytesAvailable > bytesStorable) {
- fromLim = *fromP + bytesStorable;
- output_exhausted = true;
- }
-
- /* Avoid copying partial characters (from incomplete input). */
- - const char * const fromLimBefore = fromLim;
- + fromLimBefore = fromLim;
- align_limit_to_full_utf8_characters(*fromP, &fromLim);
- if (fromLim < fromLimBefore) {
- input_incomplete = true;
- }
-
- - const ptrdiff_t bytesToCopy = fromLim - *fromP;
- + bytesToCopy = fromLim - *fromP;
- memcpy((void *)*toP, (const void *)*fromP, (size_t)bytesToCopy);
- *fromP += bytesToCopy;
- *toP += bytesToCopy;
-
-
-Create ``libexpat.lib`` (for translation) and ``libexpat.dll`` (for tests)::
-
- cl /LD *.obj libexpat.def /Felibexpat.dll
- rem for debug
- rem cl /LDd /Zi *.obj libexpat.def /Felibexpat.dll
-
- rem this will override the export library created in the step above
- rem but tests do not need the export library, they load the dll dynamically
- lib *.obj /out:libexpat.lib
-
-Then, copy
-
-- ``libexpat.lib`` into LIB
-- both ``lib\expat.h`` and ``lib\expat_external.h`` in INCLUDE
-- ``libexpat.dll`` into PATH
-
-
-The OpenSSL library
-~~~~~~~~~~~~~~~~~~~
-
-OpenSSL needs a Perl interpreter to configure its makefile. You may
-use the one distributed by ActiveState, or the one from cygwin.::
-
- svn export http://svn.python.org/projects/external/openssl-1.0.2k
- cd openssl-1.0.2k
- perl Configure VC-WIN32 no-idea no-mdc2
- ms\do_ms.bat
- nmake -f ms\nt.mak install
- copy out32\*.lib <somewhere in LIB>
- xcopy /S include\openssl <somewhere in INCLUDE>
-
-For tests you will also need the dlls::
- nmake -f ms\ntdll.mak install
- copy out32dll\*.dll <somewhere in PATH>
-
-TkInter module support
-~~~~~~~~~~~~~~~~~~~~~~
-
-Note that much of this is taken from the cpython build process.
-Tkinter is imported via cffi, so the module is optional. To recreate the tcltk
-directory found for the release script, create the dlls, libs, headers and
-runtime by running::
-
- svn export http://svn.python.org/projects/external/tcl-8.5.2.1 tcl85
- svn export http://svn.python.org/projects/external/tk-8.5.2.0 tk85
- cd tcl85\win
- nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=0 INSTALLDIR=..\..\tcltk clean all
- nmake -f makefile.vc DEBUG=0 INSTALLDIR=..\..\tcltk install
- cd ..\..\tk85\win
- nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 clean all
- nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 install
- copy ..\..\tcltk\bin\* <somewhere in PATH>
- copy ..\..\tcltk\lib\*.lib <somewhere in LIB>
- xcopy /S ..\..\tcltk\include <somewhere in INCLUDE>
-
-The lzma compression library
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Python 3.3 ship with CFFI wrappers for the lzma library, which can be
-downloaded from this site http://tukaani.org/xz. Python 3.3-3.5 use version
-5.0.5, a prebuilt version can be downloaded from
-http://tukaani.org/xz/xz-5.0.5-windows.zip, check the signature
-http://tukaani.org/xz/xz-5.0.5-windows.zip.sig
-
-Then copy the headers to the include directory, rename ``liblzma.a`` to
-``lzma.lib`` and copy it to the lib directory
-
+.. _repository: https://bitbucket.org/pypy/external
Using the mingw compiler
------------------------
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -87,7 +87,6 @@
'get_hidden_tb' : 'interp_magic.get_hidden_tb',
'lookup_special' : 'interp_magic.lookup_special',
'do_what_I_mean' : 'interp_magic.do_what_I_mean',
- 'validate_fd' : 'interp_magic.validate_fd',
'resizelist_hint' : 'interp_magic.resizelist_hint',
'newlist_hint' : 'interp_magic.newlist_hint',
'add_memory_pressure' : 'interp_magic.add_memory_pressure',
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -107,14 +107,6 @@
raise oefmt(space.w_TypeError, "expecting dict or list or set object")
return space.newtext(name)
-
- at unwrap_spec(fd='c_int')
-def validate_fd(space, fd):
- try:
- rposix.validate_fd(fd)
- except OSError as e:
- raise wrap_oserror(space, e)
-
def get_console_cp(space):
from rpython.rlib import rwin32 # Windows only
return space.newtuple([
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -30,7 +30,7 @@
from pypy.module.__builtin__.interp_classobj import W_ClassObject
from pypy.module.micronumpy.base import W_NDimArray
from rpython.rlib.entrypoint import entrypoint_lowlevel
-from rpython.rlib.rposix import is_valid_fd, validate_fd
+from rpython.rlib.rposix import FdValidator
from rpython.rlib.unroll import unrolling_iterable
from rpython.rlib.objectmodel import specialize
from pypy.module import exceptions
@@ -96,25 +96,24 @@
dash = ''
def fclose(fp):
- if not is_valid_fd(c_fileno(fp)):
+ try:
+ with FdValidator(c_fileno(fp)):
+ return c_fclose(fp)
+ except IOError:
return -1
- return c_fclose(fp)
def fwrite(buf, sz, n, fp):
- validate_fd(c_fileno(fp))
- return c_fwrite(buf, sz, n, fp)
+ with FdValidator(c_fileno(fp)):
+ return c_fwrite(buf, sz, n, fp)
def fread(buf, sz, n, fp):
- validate_fd(c_fileno(fp))
- return c_fread(buf, sz, n, fp)
+ with FdValidator(c_fileno(fp)):
+ return c_fread(buf, sz, n, fp)
_feof = rffi.llexternal('feof', [FILEP], rffi.INT)
def feof(fp):
- validate_fd(c_fileno(fp))
- return _feof(fp)
-
-def is_valid_fp(fp):
- return is_valid_fd(c_fileno(fp))
+ with FdValidator(c_fileno(fp)):
+ return _feof(fp)
pypy_decl = 'pypy_decl.h'
udir.join(pypy_decl).write("/* Will be filled later */\n")
diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py
--- a/pypy/module/cpyext/eval.py
+++ b/pypy/module/cpyext/eval.py
@@ -5,7 +5,7 @@
from rpython.rlib.rarithmetic import widen
from pypy.module.cpyext.api import (
cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, fread, feof, Py_ssize_tP,
- cpython_struct, is_valid_fp)
+ cpython_struct)
from pypy.module.cpyext.pyobject import PyObject
from pypy.module.cpyext.pyerrors import PyErr_SetFromErrno
from pypy.module.cpyext.funcobject import PyCodeObject
@@ -155,22 +155,19 @@
BUF_SIZE = 8192
source = ""
filename = rffi.charp2str(filename)
- buf = lltype.malloc(rffi.CCHARP.TO, BUF_SIZE, flavor='raw')
- if not is_valid_fp(fp):
- lltype.free(buf, flavor='raw')
- PyErr_SetFromErrno(space, space.w_IOError)
- return None
- try:
+ with rffi.scoped_alloc_buffer(BUF_SIZE) as buf:
while True:
- count = fread(buf, 1, BUF_SIZE, fp)
+ try:
+ count = fread(buf.raw, 1, BUF_SIZE, fp)
+ except OSError:
+ PyErr_SetFromErrno(space, space.w_IOError)
+ return
count = rffi.cast(lltype.Signed, count)
- source += rffi.charpsize2str(buf, count)
+ source += rffi.charpsize2str(buf.raw, count)
if count < BUF_SIZE:
if feof(fp):
break
PyErr_SetFromErrno(space, space.w_IOError)
- finally:
- lltype.free(buf, flavor='raw')
return run_string(space, source, filename, start, w_globals, w_locals)
# Undocumented function!
diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -13,7 +13,7 @@
PyEval_GetBuiltins, PyEval_GetLocals, PyEval_GetGlobals,
_PyEval_SliceIndex)
from pypy.module.cpyext.api import (
- c_fopen, c_fclose, c_fileno, Py_ssize_tP, is_valid_fd)
+ c_fopen, c_fclose, c_fileno, Py_ssize_tP)
from pypy.module.cpyext.pyobject import get_w_obj_and_decref
from pypy.interpreter.gateway import interp2app
from pypy.interpreter.error import OperationError
@@ -150,7 +150,6 @@
os.close(c_fileno(fp))
with raises_w(space, IOError):
PyRun_File(space, fp, filename, Py_file_input, w_globals, w_locals)
- if is_valid_fd(c_fileno(fp)):
c_fclose(fp)
rffi.free_charp(filename)
diff --git a/pypy/module/faulthandler/handler.py b/pypy/module/faulthandler/handler.py
--- a/pypy/module/faulthandler/handler.py
+++ b/pypy/module/faulthandler/handler.py
@@ -1,6 +1,5 @@
import os
from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
-from rpython.rlib.rposix import is_valid_fd
from rpython.rlib.rarithmetic import widen, ovfcheck_float_to_longlong
from rpython.rlib.objectmodel import keepalive_until_here
from rpython.rtyper.annlowlevel import llhelper
@@ -35,7 +34,7 @@
raise oefmt(space.w_RuntimeError, "sys.stderr is None")
elif space.isinstance_w(w_file, space.w_int):
fd = space.c_int_w(w_file)
- if fd < 0 or not is_valid_fd(fd):
+ if fd < 0:
raise oefmt(space.w_ValueError,
"file is not a valid file descriptor")
return fd, None
diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py
--- a/pypy/module/posix/app_posix.py
+++ b/pypy/module/posix/app_posix.py
@@ -1,7 +1,6 @@
# NOT_RPYTHON
from _structseq import structseqtype, structseqfield
-from __pypy__ import validate_fd
# XXX we need a way to access the current module's globals more directly...
import sys
@@ -80,25 +79,27 @@
f_flag = structseqfield(8)
f_namemax = structseqfield(9)
+# Capture file.fdopen at import time, as some code replaces
+# __builtins__.file with a custom function.
+_fdopen = file.fdopen
+
if osname == 'posix':
# POSIX: we want to check the file descriptor when fdopen() is called,
# not later when we read or write data. So we call fstat(), letting
# it raise if fd is invalid.
- _validate_fd = posix.fstat
+ def fdopen(fd, mode='r', buffering=-1):
+ """fdopen(fd [, mode='r' [, buffering]]) -> file_object
+
+ Return an open file object connected to a file descriptor."""
+ posix.fstat(fd)
+ return _fdopen(fd, mode, buffering)
+
else:
- _validate_fd = validate_fd
+ def fdopen(fd, mode='r', buffering=-1):
+ """fdopen(fd [, mode='r' [, buffering]]) -> file_object
-# Capture file.fdopen at import time, as some code replaces
-# __builtins__.file with a custom function.
-_fdopen = file.fdopen
-
-def fdopen(fd, mode='r', buffering=-1):
- """fdopen(fd [, mode='r' [, buffering]]) -> file_object
-
- Return an open file object connected to a file descriptor."""
- _validate_fd(fd)
- return _fdopen(fd, mode, buffering)
-
+ Return an open file object connected to a file descriptor."""
+ return _fdopen(fd, mode, buffering)
def tmpfile():
"""Create a temporary file.
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -18,7 +18,7 @@
USEMODULES += ['fcntl']
else:
# On windows, os.popen uses the subprocess module
- USEMODULES += ['_rawffi', 'thread', 'signal']
+ USEMODULES += ['_rawffi', 'thread', 'signal', '_cffi_backend']
def setup_module(mod):
mod.space = gettestobjspace(usemodules=USEMODULES)
@@ -300,8 +300,13 @@
# There used to be code here to ensure that fcntl is not faked
# but we can't do that cleanly any more
- exc = raises(OSError, posix.fdopen, fd)
- assert exc.value.errno == errno.EBADF
+ try:
+ fid = posix.fdopen(fd)
+ fid.read(10)
+ except IOError as e:
+ assert e.errno == errno.EBADF
+ else:
+ assert False, "using result of fdopen(fd) on closed file must raise"
def test_fdopen_hackedbuiltins(self):
"Same test, with __builtins__.file removed"
@@ -331,8 +336,17 @@
path = self.path
posix = self.posix
fd = posix.open(path, posix.O_RDONLY)
- exc = raises(OSError, posix.fdopen, fd, 'w')
- assert str(exc.value) == "[Errno 22] Invalid argument"
+ # compatability issue - using Visual Studio 10 and above no
+ # longer raises on fid creation, only when _using_ fid
+ # win32 python2 raises IOError on flush(), win32 python3 raises OSError
+ try:
+ fid = posix.fdopen(fd, 'w')
+ fid.write('abc')
+ fid.flush()
+ except (OSError, IOError) as e:
+ assert e.errno in (9, 22)
+ else:
+ assert False, "expected OSError"
posix.close(fd) # fd should not be closed
def test_getcwd(self):
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -151,9 +151,10 @@
if (rffi.TIME_T in args or rffi.TIME_TP in args
or result in (rffi.TIME_T, rffi.TIME_TP)):
name = '_' + name + '64'
+ _calling_conv = kwds.pop('calling_conv', calling_conv)
return rffi.llexternal(name, args, result,
compilation_info=eci,
- calling_conv=calling_conv,
+ calling_conv=_calling_conv,
releasegil=False,
**kwds)
@@ -187,20 +188,34 @@
"RPY_EXTERN "
"int pypy_get_daylight();\n"
"RPY_EXTERN "
- "char** pypy_get_tzname();\n"
+ "int pypy_get_tzname(size_t, int, char*);\n"
"RPY_EXTERN "
"void pypy__tzset();"],
separate_module_sources = ["""
- long pypy_get_timezone() { return timezone; }
- int pypy_get_daylight() { return daylight; }
- char** pypy_get_tzname() { return tzname; }
- void pypy__tzset() { _tzset(); }
+ long pypy_get_timezone() {
+ long timezone;
+ _get_timezone(&timezone);
+ return timezone;
+ };
+ int pypy_get_daylight() {
+ int daylight;
+ _get_daylight(&daylight);
+ return daylight;
+ };
+ int pypy_get_tzname(size_t len, int index, char * tzname) {
+ size_t s;
+ errno_t ret = _get_tzname(&s, tzname, len, index);
+ return (int)s;
+ };
+ void pypy__tzset() { _tzset(); }
"""])
# Ensure sure that we use _tzset() and timezone from the same C Runtime.
c_tzset = external('pypy__tzset', [], lltype.Void, win_eci)
c_get_timezone = external('pypy_get_timezone', [], rffi.LONG, win_eci)
c_get_daylight = external('pypy_get_daylight', [], rffi.INT, win_eci)
- c_get_tzname = external('pypy_get_tzname', [], rffi.CCHARPP, win_eci)
+ c_get_tzname = external('pypy_get_tzname',
+ [rffi.SIZE_T, rffi.INT, rffi.CCHARP],
+ rffi.INT, win_eci, calling_conv='c')
c_strftime = external('strftime', [rffi.CCHARP, rffi.SIZE_T, rffi.CCHARP, TM_P],
rffi.SIZE_T)
@@ -221,8 +236,11 @@
timezone = c_get_timezone()
altzone = timezone - 3600
daylight = c_get_daylight()
- tzname_ptr = c_get_tzname()
- tzname = rffi.charp2str(tzname_ptr[0]), rffi.charp2str(tzname_ptr[1])
+ with rffi.scoped_alloc_buffer(100) as buf:
+ s = c_get_tzname(100, 0, buf.raw)
+ tzname[0] = buf.str(s)
+ s = c_get_tzname(100, 1, buf.raw)
+ tzname[1] = buf.str(s)
if _POSIX:
if _CYGWIN:
diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py
--- a/rpython/rlib/clibffi.py
+++ b/rpython/rlib/clibffi.py
@@ -296,7 +296,8 @@
def get_libc_name():
return rwin32.GetModuleFileName(get_libc_handle())
- assert "msvcr" in get_libc_name().lower(), \
+ libc_name = get_libc_name().lower()
+ assert "msvcr" in libc_name or 'ucrtbase' in libc_name, \
"Suspect msvcrt library: %s" % (get_libc_name(),)
elif _MINGW:
def get_libc_name():
diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -173,10 +173,10 @@
def create_fdopen_rfile(fd, mode="r", buffering=-1):
newmode = _sanitize_mode(mode)
- rposix.validate_fd(fd)
ll_mode = rffi.str2charp(newmode)
try:
- ll_file = c_fdopen(fd, ll_mode)
+ with rposix.FdValidator(fd):
+ ll_file = c_fdopen(fd, ll_mode)
if not ll_file:
errno = rposix.get_saved_errno()
raise OSError(errno, os.strerror(errno))
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -39,11 +39,12 @@
if os.name == 'nt':
if platform.name == 'msvc':
- includes=['errno.h','stdio.h']
+ includes=['errno.h','stdio.h', 'stdlib.h']
else:
includes=['errno.h','stdio.h', 'stdint.h']
separate_module_sources =['''
- /* Lifted completely from CPython 3.3 Modules/posix_module.c */
+ /* Lifted completely from CPython 3 Modules/posixmodule.c */
+ #if defined _MSC_VER && _MSC_VER >= 1400 && _MSC_VER < 1900
#include <malloc.h> /* for _msize */
typedef struct {
intptr_t osfhnd;
@@ -95,6 +96,46 @@
errno = EBADF;
return 0;
}
+ RPY_EXTERN void* enter_suppress_iph(void) {return (void*)NULL;};
+ RPY_EXTERN void exit_suppress_iph(void* handle) {};
+ #elif defined _MSC_VER
+ RPY_EXTERN int _PyVerify_fd(int fd)
+ {
+ return 1;
+ }
+ static void __cdecl _Py_silent_invalid_parameter_handler(
+ wchar_t const* expression,
+ wchar_t const* function,
+ wchar_t const* file,
+ unsigned int line,
+ uintptr_t pReserved) {
+ wprintf(L"Invalid parameter detected in function %s."
+ L" File: %s Line: %d\\n", function, file, line);
+ wprintf(L"Expression: %s\\n", expression);
+ }
+
+ RPY_EXTERN void* enter_suppress_iph(void)
+ {
+ void* ret = _set_thread_local_invalid_parameter_handler(_Py_silent_invalid_parameter_handler);
+ /*fprintf(stdout, "setting %p returning %p\\n", (void*)_Py_silent_invalid_parameter_handler, ret);*/
+ return ret;
+ }
+ RPY_EXTERN void exit_suppress_iph(void* old_handler)
+ {
+ void * ret;
+ _invalid_parameter_handler _handler = (_invalid_parameter_handler)old_handler;
+ ret = _set_thread_local_invalid_parameter_handler(_handler);
+ /*fprintf(stdout, "exiting, setting %p returning %p\\n", old_handler, ret);*/
+ }
+
+ #else
+ RPY_EXTERN int _PyVerify_fd(int fd)
+ {
+ return 1;
+ }
+ RPY_EXTERN void* enter_suppress_iph(void) {return (void*)NULL;};
+ RPY_EXTERN void exit_suppress_iph(void* handle) {};
+ #endif
''',]
post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);']
else:
@@ -193,50 +234,6 @@
else:
rthread.tlfield_rpy_errno.setraw(_get_errno())
# ^^^ keep fork() up-to-date too, below
-
-
-if os.name == 'nt':
- is_valid_fd = jit.dont_look_inside(rffi.llexternal(
- "_PyVerify_fd", [rffi.INT], rffi.INT,
- compilation_info=errno_eci,
- ))
- @enforceargs(int)
- def validate_fd(fd):
- if not is_valid_fd(fd):
- from errno import EBADF
- raise OSError(EBADF, 'Bad file descriptor')
-
- def _bound_for_write(fd, count):
- if count > 32767 and c_isatty(fd):
- # CPython Issue #11395, PyPy Issue #2636: the Windows console
- # returns an error (12: not enough space error) on writing into
- # stdout if stdout mode is binary and the length is greater than
- # 66,000 bytes (or less, depending on heap usage). Can't easily
- # test that, because we need 'fd' to be non-redirected...
- count = 32767
- elif count > 0x7fffffff:
- count = 0x7fffffff
- return count
-else:
- def is_valid_fd(fd):
- return 1
-
- @enforceargs(int)
- def validate_fd(fd):
- pass
-
- def _bound_for_write(fd, count):
- return count
-
-def closerange(fd_low, fd_high):
- # this behaves like os.closerange() from Python 2.6.
- for fd in xrange(fd_low, fd_high):
- try:
- if is_valid_fd(fd):
- os.close(fd)
- except OSError:
- pass
-
if _WIN32:
includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h']
libraries = []
@@ -258,11 +255,80 @@
if sys.platform.startswith('freebsd') or sys.platform.startswith('openbsd'):
includes.append('sys/ttycom.h')
libraries = ['util']
+
eci = ExternalCompilationInfo(
includes=includes,
libraries=libraries,
)
+def external(name, args, result, compilation_info=eci, **kwds):
+ return rffi.llexternal(name, args, result,
+ compilation_info=compilation_info, **kwds)
+
+
+if os.name == 'nt':
+ is_valid_fd = jit.dont_look_inside(external("_PyVerify_fd", [rffi.INT],
+ rffi.INT, compilation_info=errno_eci,
+ ))
+ c_enter_suppress_iph = jit.dont_look_inside(external("enter_suppress_iph",
+ [], rffi.VOIDP, compilation_info=errno_eci))
+ c_exit_suppress_iph = jit.dont_look_inside(external("exit_suppress_iph",
+ [rffi.VOIDP], lltype.Void,
+ compilation_info=errno_eci))
+
+ @enforceargs(int)
+ def _validate_fd(fd):
+ if not is_valid_fd(fd):
+ from errno import EBADF
+ raise OSError(EBADF, 'Bad file descriptor')
+
+ class FdValidator(object):
+
+ def __init__(self, fd):
+ _validate_fd(fd)
+
+ def __enter__(self):
+ self.invalid_param_hndlr = c_enter_suppress_iph()
+ return self
+
+ def __exit__(self, *args):
+ c_exit_suppress_iph(self.invalid_param_hndlr)
+
+ def _bound_for_write(fd, count):
+ if count > 32767 and c_isatty(fd):
+ # CPython Issue #11395, PyPy Issue #2636: the Windows console
+ # returns an error (12: not enough space error) on writing into
+ # stdout if stdout mode is binary and the length is greater than
+ # 66,000 bytes (or less, depending on heap usage). Can't easily
+ # test that, because we need 'fd' to be non-redirected...
+ count = 32767
+ elif count > 0x7fffffff:
+ count = 0x7fffffff
+ return count
+else:
+ class FdValidator(object):
+
+ def __init__(self, fd):
+ pass
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args):
+ pass
+
+ def _bound_for_write(fd, count):
+ return count
+
+def closerange(fd_low, fd_high):
+ # this behaves like os.closerange() from Python 2.6.
+ for fd in xrange(fd_low, fd_high):
+ try:
+ with FdValidator(fd):
+ os.close(fd)
+ except OSError:
+ pass
+
class CConfig:
_compilation_info_ = eci
SEEK_SET = rffi_platform.DefinedConstantInteger('SEEK_SET')
@@ -315,10 +381,6 @@
config = rffi_platform.configure(CConfig)
globals().update(config)
-def external(name, args, result, compilation_info=eci, **kwds):
- return rffi.llexternal(name, args, result,
- compilation_info=compilation_info, **kwds)
-
# For now we require off_t to be the same size as LONGLONG, which is the
# interface required by callers of functions that thake an argument of type
# off_t.
@@ -410,12 +472,12 @@
return result
def _dup(fd, inheritable=True):
- validate_fd(fd)
- if inheritable:
- res = c_dup(fd)
- else:
- res = c_dup_noninheritable(fd)
- return res
+ with FdValidator(fd):
+ if inheritable:
+ res = c_dup(fd)
+ else:
+ res = c_dup_noninheritable(fd)
+ return res
@replace_os_function('dup')
def dup(fd, inheritable=True):
@@ -424,12 +486,12 @@
@replace_os_function('dup2')
def dup2(fd, newfd, inheritable=True):
- validate_fd(fd)
- if inheritable:
- res = c_dup2(fd, newfd)
- else:
- res = c_dup2_noninheritable(fd, newfd)
- handle_posix_error('dup2', res)
+ with FdValidator(fd):
+ if inheritable:
+ res = c_dup2(fd, newfd)
+ else:
+ res = c_dup2_noninheritable(fd, newfd)
+ handle_posix_error('dup2', res)
#___________________________________________________________________
@@ -457,25 +519,26 @@
def read(fd, count):
if count < 0:
raise OSError(errno.EINVAL, None)
- validate_fd(fd)
- with rffi.scoped_alloc_buffer(count) as buf:
- void_buf = rffi.cast(rffi.VOIDP, buf.raw)
- got = handle_posix_error('read', c_read(fd, void_buf, count))
- return buf.str(got)
+ with FdValidator(fd):
+ with rffi.scoped_alloc_buffer(count) as buf:
+ void_buf = rffi.cast(rffi.VOIDP, buf.raw)
+ got = handle_posix_error('read', c_read(fd, void_buf, count))
+ return buf.str(got)
@replace_os_function('write')
@enforceargs(int, None)
def write(fd, data):
count = len(data)
- validate_fd(fd)
- count = _bound_for_write(fd, count)
- with rffi.scoped_nonmovingbuffer(data) as buf:
- return handle_posix_error('write', c_write(fd, buf, count))
+ with FdValidator(fd):
+ count = _bound_for_write(fd, count)
+ with rffi.scoped_nonmovingbuffer(data) as buf:
+ ret = c_write(fd, buf, count)
+ return handle_posix_error('write', ret)
@replace_os_function('close')
def close(fd):
- validate_fd(fd)
- handle_posix_error('close', c_close(fd))
+ with FdValidator(fd):
+ handle_posix_error('close', c_close(fd))
c_lseek = external('_lseeki64' if _WIN32 else 'lseek',
[rffi.INT, rffi.LONGLONG, rffi.INT], rffi.LONGLONG,
@@ -483,15 +546,15 @@
@replace_os_function('lseek')
def lseek(fd, pos, how):
- validate_fd(fd)
- if SEEK_SET is not None:
- if how == 0:
- how = SEEK_SET
- elif how == 1:
- how = SEEK_CUR
- elif how == 2:
- how = SEEK_END
- return handle_posix_error('lseek', c_lseek(fd, pos, how))
+ with FdValidator(fd):
+ if SEEK_SET is not None:
+ if how == 0:
+ how = SEEK_SET
+ elif how == 1:
+ how = SEEK_CUR
+ elif how == 2:
+ how = SEEK_END
+ return handle_posix_error('lseek', c_lseek(fd, pos, how))
if not _WIN32:
c_pread = external('pread',
@@ -505,7 +568,6 @@
def pread(fd, count, offset):
if count < 0:
raise OSError(errno.EINVAL, None)
- validate_fd(fd)
with rffi.scoped_alloc_buffer(count) as buf:
void_buf = rffi.cast(rffi.VOIDP, buf.raw)
return buf.str(handle_posix_error('pread', c_pread(fd, void_buf, count, offset)))
@@ -513,7 +575,6 @@
@enforceargs(int, None, None)
def pwrite(fd, data, offset):
count = len(data)
- validate_fd(fd)
with rffi.scoped_nonmovingbuffer(data) as buf:
return handle_posix_error('pwrite', c_pwrite(fd, buf, count, offset))
@@ -524,7 +585,6 @@
@enforceargs(int, None, None)
def posix_fallocate(fd, offset, length):
- validate_fd(fd)
return handle_posix_error('posix_fallocate', c_posix_fallocate(fd, offset, length))
if HAVE_FADVISE:
@@ -546,7 +606,6 @@
@enforceargs(int, None, None, int)
def posix_fadvise(fd, offset, length, advice):
- validate_fd(fd)
error = c_posix_fadvise(fd, offset, length, advice)
error = widen(error)
if error != 0:
@@ -557,7 +616,6 @@
save_err=rffi.RFFI_SAVE_ERRNO)
@enforceargs(int, None, None)
def lockf(fd, cmd, length):
- validate_fd(fd)
return handle_posix_error('lockf', c_lockf(fd, cmd, length))
c_ftruncate = external('ftruncate', [rffi.INT, rffi.LONGLONG], rffi.INT,
@@ -571,18 +629,18 @@
@replace_os_function('ftruncate')
def ftruncate(fd, length):
- validate_fd(fd)
- handle_posix_error('ftruncate', c_ftruncate(fd, length))
+ with FdValidator(fd):
+ handle_posix_error('ftruncate', c_ftruncate(fd, length))
@replace_os_function('fsync')
def fsync(fd):
- validate_fd(fd)
- handle_posix_error('fsync', c_fsync(fd))
+ with FdValidator(fd):
+ handle_posix_error('fsync', c_fsync(fd))
@replace_os_function('fdatasync')
def fdatasync(fd):
- validate_fd(fd)
- handle_posix_error('fdatasync', c_fdatasync(fd))
+ with FdValidator(fd):
+ handle_posix_error('fdatasync', c_fdatasync(fd))
def sync():
c_sync()
@@ -649,8 +707,8 @@
@replace_os_function('fchdir')
def fchdir(fd):
- validate_fd(fd)
- handle_posix_error('fchdir', c_fchdir(fd))
+ with FdValidator(fd):
+ handle_posix_error('fchdir', c_fchdir(fd))
@replace_os_function('access')
@specialize.argtype(0)
@@ -1090,9 +1148,9 @@
@replace_os_function('isatty')
def isatty(fd):
- if not is_valid_fd(fd):
- return False
- return c_isatty(fd) != 0
+ with FdValidator(fd):
+ return c_isatty(fd) != 0
+ return False
c_ttyname = external('ttyname', [lltype.Signed], rffi.CCHARP,
releasegil=False,
diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -117,7 +117,7 @@
from rpython.translator.platform import host_factory
static_platform = host_factory()
if static_platform.name == 'msvc':
- defines += ' PROCESS_QUERY_LIMITED_INFORMATION'
+ defines += ' PROCESS_QUERY_LIMITED_INFORMATION'
for name in defines.split():
locals()[name] = rffi_platform.ConstantInteger(name)
@@ -171,7 +171,7 @@
function, if that C function was declared with the flag
llexternal(..., save_err=RFFI_SAVE_LASTERROR | RFFI_ALT_ERRNO).
Functions without that flag don't change the saved LastError.
- Alternatively, if the function was declared
+ Alternatively, if the function was declared
RFFI_SAVE_WSALASTERROR | RFFI_ALT_ERRNO,
then the value of the C-level WSAGetLastError() is saved instead
(into the same "saved alt LastError" variable).
@@ -218,9 +218,9 @@
_get_osfhandle = rffi.llexternal('_get_osfhandle', [rffi.INT], HANDLE)
def get_osfhandle(fd):
- from rpython.rlib.rposix import validate_fd
- validate_fd(fd)
- handle = _get_osfhandle(fd)
+ from rpython.rlib.rposix import FdValidator
+ with FdValidator(fd):
+ handle = _get_osfhandle(fd)
if handle == INVALID_HANDLE_VALUE:
raise WindowsError(ERROR_INVALID_HANDLE, "Invalid file handle")
return handle
@@ -231,45 +231,10 @@
in the dict."""
# Prior to Visual Studio 8, the MSVCRT dll doesn't export the
# _dosmaperr() function, which is available only when compiled
- # against the static CRT library.
- from rpython.translator.platform import host_factory
- static_platform = host_factory()
- if static_platform.name == 'msvc':
- static_platform.cflags = ['/MT'] # static CRT
- static_platform.version = 0 # no manifest
- cfile = udir.join('dosmaperr.c')
- cfile.write(r'''
- #include <errno.h>
- #include <WinError.h>
- #include <stdio.h>
- #ifdef __GNUC__
- #define _dosmaperr mingw_dosmaperr
- #endif
- int main()
- {
- int i;
- for(i=1; i < 65000; i++) {
- _dosmaperr(i);
- if (errno == EINVAL) {
- /* CPython issue #12802 */
- if (i == ERROR_DIRECTORY)
- errno = ENOTDIR;
- else
- continue;
- }
- printf("%d\t%d\n", i, errno);
- }
- return 0;
- }''')
- try:
- exename = static_platform.compile(
- [cfile], ExternalCompilationInfo(),
- outputfilename = "dosmaperr",
- standalone=True)
- except (CompilationError, WindowsError):
- # Fallback for the mingw32 compiler
- assert static_platform.name == 'mingw32'
- errors = {
+ # against the static CRT library. After Visual Studio 9, this
+ # private function seems to be gone, so use a static map, from
+ # CPython PC/errmap.h
+ errors = {
2: 2, 3: 2, 4: 24, 5: 13, 6: 9, 7: 12, 8: 12, 9: 12, 10: 7,
11: 8, 15: 2, 16: 13, 17: 18, 18: 2, 19: 13, 20: 13, 21: 13,
22: 13, 23: 13, 24: 13, 25: 13, 26: 13, 27: 13, 28: 13,
@@ -279,12 +244,8 @@
132: 13, 145: 41, 158: 13, 161: 2, 164: 11, 167: 13, 183: 17,
188: 8, 189: 8, 190: 8, 191: 8, 192: 8, 193: 8, 194: 8,
195: 8, 196: 8, 197: 8, 198: 8, 199: 8, 200: 8, 201: 8,
- 202: 8, 206: 2, 215: 11, 267: 20, 1816: 12,
+ 202: 8, 206: 2, 215: 11, 232: 32, 267: 20, 1816: 12,
}
- else:
- output = os.popen(str(exename))
- errors = dict(map(int, line.split())
- for line in output)
return errors, errno.EINVAL
# A bit like strerror...
@@ -299,7 +260,7 @@
buf[0] = lltype.nullptr(rffi.CCHARP.TO)
try:
msglen = FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER |
- FORMAT_MESSAGE_FROM_SYSTEM |
+ FORMAT_MESSAGE_FROM_SYSTEM |
FORMAT_MESSAGE_IGNORE_INSERTS,
None,
rffi.cast(DWORD, code),
@@ -330,7 +291,7 @@
buf[0] = lltype.nullptr(rffi.CWCHARP.TO)
try:
msglen = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER |
- FORMAT_MESSAGE_FROM_SYSTEM |
+ FORMAT_MESSAGE_FROM_SYSTEM |
FORMAT_MESSAGE_IGNORE_INSERTS,
None,
rffi.cast(DWORD, code),
diff --git a/rpython/rlib/test/test_rwin32.py b/rpython/rlib/test/test_rwin32.py
--- a/rpython/rlib/test/test_rwin32.py
+++ b/rpython/rlib/test/test_rwin32.py
@@ -15,19 +15,6 @@
py.test.raises(OSError, rwin32.get_osfhandle, fd)
rwin32.get_osfhandle(0)
-def test_get_osfhandle_raising():
- #try to test what kind of exception get_osfhandle raises w/out fd validation
- py.test.skip('Crashes python')
- fid = open(str(udir.join('validate_test.txt')), 'w')
- fd = fid.fileno()
- fid.close()
- def validate_fd(fd):
- return 1
- _validate_fd = rwin32.validate_fd
- rwin32.validate_fd = validate_fd
- raises(WindowsError, rwin32.get_osfhandle, fd)
- rwin32.validate_fd = _validate_fd
-
def test_open_process():
pid = rwin32.GetCurrentProcessId()
assert pid != 0
diff --git a/rpython/tool/setuptools_msvc.py b/rpython/tool/setuptools_msvc.py
new file mode 100644
--- /dev/null
+++ b/rpython/tool/setuptools_msvc.py
@@ -0,0 +1,1308 @@
+"""
+Improved support for Microsoft Visual C++ compilers.
+
+Known supported compilers:
+--------------------------
+Microsoft Visual C++ 9.0:
+ Microsoft Visual C++ Compiler for Python 2.7 (x86, amd64)
+ Microsoft Windows SDK 6.1 (x86, x64, ia64)
+ Microsoft Windows SDK 7.0 (x86, x64, ia64)
+
+Microsoft Visual C++ 10.0:
+ Microsoft Windows SDK 7.1 (x86, x64, ia64)
+
+Microsoft Visual C++ 14.0:
+ Microsoft Visual C++ Build Tools 2015 (x86, x64, arm)
+ Microsoft Visual Studio 2017 (x86, x64, arm, arm64)
+ Microsoft Visual Studio Build Tools 2017 (x86, x64, arm, arm64)
+"""
+
+# copied as-is from python3
+# modified to:
+# - commented out monkey patching
+# - lookup() returns str not unicode
+
+import os
+import sys
+import platform
+import itertools
+import distutils.errors
+from pkg_resources.extern.packaging.version import LegacyVersion
+
+from setuptools.extern.six.moves import filterfalse
+
+#from .monkey import get_unpatched
+
+if platform.system() == 'Windows':
+ from setuptools.extern.six.moves import winreg
+ safe_env = os.environ
+else:
+ """
+ Mock winreg and environ so the module can be imported
+ on this platform.
+ """
+
+ class winreg:
+ HKEY_USERS = None
+ HKEY_CURRENT_USER = None
+ HKEY_LOCAL_MACHINE = None
+ HKEY_CLASSES_ROOT = None
+
+ safe_env = dict()
+
+_msvc9_suppress_errors = (
+ # msvc9compiler isn't available on some platforms
+ ImportError,
+
+ # msvc9compiler raises DistutilsPlatformError in some
+ # environments. See #1118.
+ distutils.errors.DistutilsPlatformError,
+)
+
+try:
+ from distutils.msvc9compiler import Reg
+except _msvc9_suppress_errors:
+ pass
+
+
+def msvc9_find_vcvarsall(version):
+ """
+ Patched "distutils.msvc9compiler.find_vcvarsall" to use the standalone
+ compiler build for Python (VCForPython). Fall back to original behavior
+ when the standalone compiler is not available.
+
+ Redirect the path of "vcvarsall.bat".
+
+ Known supported compilers
+ -------------------------
+ Microsoft Visual C++ 9.0:
+ Microsoft Visual C++ Compiler for Python 2.7 (x86, amd64)
+
+ Parameters
+ ----------
+ version: float
+ Required Microsoft Visual C++ version.
+
+ Return
+ ------
+ vcvarsall.bat path: str
+ """
+ VC_BASE = r'Software\%sMicrosoft\DevDiv\VCForPython\%0.1f'
+ key = VC_BASE % ('', version)
+ try:
+ # Per-user installs register the compiler path here
+ productdir = Reg.get_value(key, "installdir")
+ except KeyError:
+ try:
+ # All-user installs on a 64-bit system register here
+ key = VC_BASE % ('Wow6432Node\\', version)
+ productdir = Reg.get_value(key, "installdir")
+ except KeyError:
+ productdir = None
+
+ if productdir:
+ vcvarsall = os.path.os.path.join(productdir, "vcvarsall.bat")
+ if os.path.isfile(vcvarsall):
+ return vcvarsall
+
+ return get_unpatched(msvc9_find_vcvarsall)(version)
+
+
+def msvc9_query_vcvarsall(ver, arch='x86', *args, **kwargs):
+ """
+ Patched "distutils.msvc9compiler.query_vcvarsall" for support extra
+ compilers.
+
+ Set environment without use of "vcvarsall.bat".
+
+ Known supported compilers
+ -------------------------
+ Microsoft Visual C++ 9.0:
+ Microsoft Visual C++ Compiler for Python 2.7 (x86, amd64)
+ Microsoft Windows SDK 6.1 (x86, x64, ia64)
+ Microsoft Windows SDK 7.0 (x86, x64, ia64)
+
+ Microsoft Visual C++ 10.0:
+ Microsoft Windows SDK 7.1 (x86, x64, ia64)
+
+ Parameters
+ ----------
+ ver: float
+ Required Microsoft Visual C++ version.
+ arch: str
+ Target architecture.
+
+ Return
+ ------
+ environment: dict
+ """
+ # Try to get environement from vcvarsall.bat (Classical way)
+ #try:
+ # orig = get_unpatched(msvc9_query_vcvarsall)
+ # return orig(ver, arch, *args, **kwargs)
+ #except distutils.errors.DistutilsPlatformError:
+ # # Pass error if Vcvarsall.bat is missing
+ # pass
+ #except ValueError:
+ # # Pass error if environment not set after executing vcvarsall.bat
+ # pass
+
+ # If error, try to set environment directly
+ try:
+ return EnvironmentInfo(arch, ver).return_env()
+ except distutils.errors.DistutilsPlatformError as exc:
+ _augment_exception(exc, ver, arch)
+ raise
+
+
+def msvc14_get_vc_env(plat_spec):
+ """
+ Patched "distutils._msvccompiler._get_vc_env" for support extra
+ compilers.
+
+ Set environment without use of "vcvarsall.bat".
+
+ Known supported compilers
+ -------------------------
+ Microsoft Visual C++ 14.0:
+ Microsoft Visual C++ Build Tools 2015 (x86, x64, arm)
+ Microsoft Visual Studio 2017 (x86, x64, arm, arm64)
+ Microsoft Visual Studio Build Tools 2017 (x86, x64, arm, arm64)
+
+ Parameters
+ ----------
+ plat_spec: str
+ Target architecture.
+
+ Return
+ ------
+ environment: dict
+ """
+ # Try to get environment from vcvarsall.bat (Classical way)
+ #try:
+ # return get_unpatched(msvc14_get_vc_env)(plat_spec)
+ #except distutils.errors.DistutilsPlatformError:
+ # Pass error Vcvarsall.bat is missing
+ # pass
+
+ # If error, try to set environment directly
+ try:
+ return EnvironmentInfo(plat_spec, vc_min_ver=14.0).return_env()
+ except distutils.errors.DistutilsPlatformError as exc:
+ _augment_exception(exc, 14.0)
+ raise
+
+
+def msvc14_gen_lib_options(*args, **kwargs):
+ """
+ Patched "distutils._msvccompiler.gen_lib_options" for fix
+ compatibility between "numpy.distutils" and "distutils._msvccompiler"
+ (for Numpy < 1.11.2)
+ """
+ if "numpy.distutils" in sys.modules:
+ import numpy as np
+ if LegacyVersion(np.__version__) < LegacyVersion('1.11.2'):
+ return np.distutils.ccompiler.gen_lib_options(*args, **kwargs)
+ return get_unpatched(msvc14_gen_lib_options)(*args, **kwargs)
+
+
+def _augment_exception(exc, version, arch=''):
+ """
+ Add details to the exception message to help guide the user
+ as to what action will resolve it.
+ """
+ # Error if MSVC++ directory not found or environment not set
+ message = exc.args[0]
+
+ if "vcvarsall" in message.lower() or "visual c" in message.lower():
+ # Special error message if MSVC++ not installed
+ tmpl = 'Microsoft Visual C++ {version:0.1f} is required.'
+ message = tmpl.format(**locals())
+ msdownload = 'www.microsoft.com/download/details.aspx?id=%d'
+ if version == 9.0:
+ if arch.lower().find('ia64') > -1:
+ # For VC++ 9.0, if IA64 support is needed, redirect user
+ # to Windows SDK 7.0
+ message += ' Get it with "Microsoft Windows SDK 7.0": '
+ message += msdownload % 3138
+ else:
+ # For VC++ 9.0 redirect user to Vc++ for Python 2.7 :
+ # This redirection link is maintained by Microsoft.
+ # Contact vspython at microsoft.com if it needs updating.
+ message += ' Get it from http://aka.ms/vcpython27'
+ elif version == 10.0:
+ # For VC++ 10.0 Redirect user to Windows SDK 7.1
+ message += ' Get it with "Microsoft Windows SDK 7.1": '
+ message += msdownload % 8279
+ elif version >= 14.0:
+ # For VC++ 14.0 Redirect user to Visual C++ Build Tools
+ message += (' Get it with "Microsoft Visual C++ Build Tools": '
+ r'http://landinghub.visualstudio.com/'
+ 'visual-cpp-build-tools')
+
+ exc.args = (message, )
+
+
+class PlatformInfo:
+ """
+ Current and Target Architectures informations.
+
+ Parameters
+ ----------
+ arch: str
+ Target architecture.
+ """
+ current_cpu = safe_env.get('processor_architecture', '').lower()
+
+ def __init__(self, arch):
+ self.arch = arch.lower().replace('x64', 'amd64')
+
+ @property
+ def target_cpu(self):
+ return self.arch[self.arch.find('_') + 1:]
+
+ def target_is_x86(self):
+ return self.target_cpu == 'x86'
+
+ def current_is_x86(self):
+ return self.current_cpu == 'x86'
+
+ def current_dir(self, hidex86=False, x64=False):
+ """
+ Current platform specific subfolder.
+
+ Parameters
+ ----------
+ hidex86: bool
+ return '' and not '\x86' if architecture is x86.
+ x64: bool
+ return '\x64' and not '\amd64' if architecture is amd64.
+
+ Return
+ ------
+ subfolder: str
+ '\target', or '' (see hidex86 parameter)
+ """
+ return (
+ '' if (self.current_cpu == 'x86' and hidex86) else
+ r'\x64' if (self.current_cpu == 'amd64' and x64) else
+ r'\%s' % self.current_cpu
+ )
+
+ def target_dir(self, hidex86=False, x64=False):
+ r"""
+ Target platform specific subfolder.
+
+ Parameters
+ ----------
+ hidex86: bool
+ return '' and not '\x86' if architecture is x86.
+ x64: bool
+ return '\x64' and not '\amd64' if architecture is amd64.
+
+ Return
+ ------
+ subfolder: str
+ '\current', or '' (see hidex86 parameter)
+ """
+ return (
+ '' if (self.target_cpu == 'x86' and hidex86) else
+ r'\x64' if (self.target_cpu == 'amd64' and x64) else
+ r'\%s' % self.target_cpu
+ )
+
+ def cross_dir(self, forcex86=False):
+ r"""
+ Cross platform specific subfolder.
+
+ Parameters
+ ----------
+ forcex86: bool
+ Use 'x86' as current architecture even if current acritecture is
+ not x86.
+
+ Return
+ ------
+ subfolder: str
+ '' if target architecture is current architecture,
+ '\current_target' if not.
+ """
+ current = 'x86' if forcex86 else self.current_cpu
+ return (
+ '' if self.target_cpu == current else
+ self.target_dir().replace('\\', '\\%s_' % current)
+ )
+
+
+class RegistryInfo:
+ """
+ Microsoft Visual Studio related registry informations.
+
+ Parameters
+ ----------
+ platform_info: PlatformInfo
+ "PlatformInfo" instance.
+ """
+ HKEYS = (winreg.HKEY_USERS,
+ winreg.HKEY_CURRENT_USER,
+ winreg.HKEY_LOCAL_MACHINE,
+ winreg.HKEY_CLASSES_ROOT)
+
+ def __init__(self, platform_info):
+ self.pi = platform_info
+
+ @property
+ def visualstudio(self):
+ """
+ Microsoft Visual Studio root registry key.
+ """
+ return 'VisualStudio'
+
+ @property
+ def sxs(self):
+ """
+ Microsoft Visual Studio SxS registry key.
+ """
+ return os.path.join(self.visualstudio, 'SxS')
+
+ @property
+ def vc(self):
+ """
+ Microsoft Visual C++ VC7 registry key.
+ """
+ return os.path.join(self.sxs, 'VC7')
+
+ @property
+ def vs(self):
+ """
+ Microsoft Visual Studio VS7 registry key.
+ """
+ return os.path.join(self.sxs, 'VS7')
+
+ @property
+ def vc_for_python(self):
+ """
+ Microsoft Visual C++ for Python registry key.
+ """
+ return r'DevDiv\VCForPython'
+
+ @property
+ def microsoft_sdk(self):
+ """
+ Microsoft SDK registry key.
+ """
+ return 'Microsoft SDKs'
+
+ @property
+ def windows_sdk(self):
+ """
+ Microsoft Windows/Platform SDK registry key.
+ """
+ return os.path.join(self.microsoft_sdk, 'Windows')
+
+ @property
+ def netfx_sdk(self):
+ """
+ Microsoft .NET Framework SDK registry key.
+ """
+ return os.path.join(self.microsoft_sdk, 'NETFXSDK')
+
+ @property
+ def windows_kits_roots(self):
+ """
+ Microsoft Windows Kits Roots registry key.
+ """
+ return r'Windows Kits\Installed Roots'
+
+ def microsoft(self, key, x86=False):
+ """
+ Return key in Microsoft software registry.
+
+ Parameters
+ ----------
+ key: str
+ Registry key path where look.
+ x86: str
+ Force x86 software registry.
+
+ Return
+ ------
+ str: value
+ """
+ node64 = '' if self.pi.current_is_x86() or x86 else 'Wow6432Node'
+ return os.path.join('Software', node64, 'Microsoft', key)
+
+ def lookup(self, key, name):
+ """
+ Look for values in registry in Microsoft software registry.
+
+ Parameters
+ ----------
+ key: str
+ Registry key path where look.
+ name: str
+ Value name to find.
+
+ Return
+ ------
+ str: value
+ """
+ KEY_READ = winreg.KEY_READ
+ openkey = winreg.OpenKey
+ ms = self.microsoft
+ for hkey in self.HKEYS:
+ try:
+ bkey = openkey(hkey, ms(key), 0, KEY_READ)
+ except (OSError, IOError):
+ if not self.pi.current_is_x86():
+ try:
+ bkey = openkey(hkey, ms(key, True), 0, KEY_READ)
+ except (OSError, IOError):
+ continue
+ else:
+ continue
+ try:
+ # modified to return str for python2
+ return winreg.QueryValueEx(bkey, name)[0].encode('utf8')
+ except (OSError, IOError):
+ pass
+
+
+class SystemInfo:
+ """
+ Microsoft Windows and Visual Studio related system inormations.
+
+ Parameters
+ ----------
+ registry_info: RegistryInfo
+ "RegistryInfo" instance.
+ vc_ver: float
+ Required Microsoft Visual C++ version.
+ """
+
+ # Variables and properties in this class use originals CamelCase variables
+ # names from Microsoft source files for more easy comparaison.
+ WinDir = safe_env.get('WinDir', '')
+ ProgramFiles = safe_env.get('ProgramFiles', '')
+ ProgramFilesx86 = safe_env.get('ProgramFiles(x86)', ProgramFiles)
+
+ def __init__(self, registry_info, vc_ver=None):
+ self.ri = registry_info
+ self.pi = self.ri.pi
+ self.vc_ver = vc_ver or self._find_latest_available_vc_ver()
+
+ def _find_latest_available_vc_ver(self):
+ try:
+ return self.find_available_vc_vers()[-1]
+ except IndexError:
+ err = 'No Microsoft Visual C++ version found'
+ raise distutils.errors.DistutilsPlatformError(err)
+
+ def find_available_vc_vers(self):
+ """
+ Find all available Microsoft Visual C++ versions.
+ """
+ ms = self.ri.microsoft
+ vckeys = (self.ri.vc, self.ri.vc_for_python, self.ri.vs)
+ vc_vers = []
+ for hkey in self.ri.HKEYS:
+ for key in vckeys:
+ try:
+ bkey = winreg.OpenKey(hkey, ms(key), 0, winreg.KEY_READ)
+ except (OSError, IOError):
+ continue
+ subkeys, values, _ = winreg.QueryInfoKey(bkey)
+ for i in range(values):
+ try:
+ ver = float(winreg.EnumValue(bkey, i)[0])
+ if ver not in vc_vers:
+ vc_vers.append(ver)
+ except ValueError:
+ pass
+ for i in range(subkeys):
+ try:
+ ver = float(winreg.EnumKey(bkey, i))
+ if ver not in vc_vers:
+ vc_vers.append(ver)
+ except ValueError:
+ pass
+ return sorted(vc_vers)
+
+ @property
+ def VSInstallDir(self):
+ """
+ Microsoft Visual Studio directory.
+ """
+ # Default path
+ name = 'Microsoft Visual Studio %0.1f' % self.vc_ver
+ default = os.path.join(self.ProgramFilesx86, name)
+
+ # Try to get path from registry, if fail use default path
+ return self.ri.lookup(self.ri.vs, '%0.1f' % self.vc_ver) or default
+
+ @property
+ def VCInstallDir(self):
+ """
+ Microsoft Visual C++ directory.
+ """
+ self.VSInstallDir
+
+ guess_vc = self._guess_vc() or self._guess_vc_legacy()
+
+ # Try to get "VC++ for Python" path from registry as default path
+ reg_path = os.path.join(self.ri.vc_for_python, '%0.1f' % self.vc_ver)
+ python_vc = self.ri.lookup(reg_path, 'installdir')
+ default_vc = os.path.join(python_vc, 'VC') if python_vc else guess_vc
+
+ # Try to get path from registry, if fail use default path
+ path = self.ri.lookup(self.ri.vc, '%0.1f' % self.vc_ver) or default_vc
+
+ if not os.path.isdir(path):
+ msg = 'Microsoft Visual C++ directory not found'
+ raise distutils.errors.DistutilsPlatformError(msg)
+
+ return path
+
+ def _guess_vc(self):
+ """
+ Locate Visual C for 2017
+ """
+ if self.vc_ver <= 14.0:
+ return
+
+ default = r'VC\Tools\MSVC'
+ guess_vc = os.path.join(self.VSInstallDir, default)
+ # Subdir with VC exact version as name
+ try:
+ vc_exact_ver = os.listdir(guess_vc)[-1]
+ return os.path.join(guess_vc, vc_exact_ver)
+ except (OSError, IOError, IndexError):
+ pass
+
+ def _guess_vc_legacy(self):
+ """
+ Locate Visual C for versions prior to 2017
+ """
+ default = r'Microsoft Visual Studio %0.1f\VC' % self.vc_ver
+ return os.path.join(self.ProgramFilesx86, default)
+
+ @property
+ def WindowsSdkVersion(self):
+ """
+ Microsoft Windows SDK versions for specified MSVC++ version.
+ """
+ if self.vc_ver <= 9.0:
+ return ('7.0', '6.1', '6.0a')
+ elif self.vc_ver == 10.0:
+ return ('7.1', '7.0a')
+ elif self.vc_ver == 11.0:
+ return ('8.0', '8.0a')
+ elif self.vc_ver == 12.0:
+ return ('8.1', '8.1a')
+ elif self.vc_ver >= 14.0:
+ return ('10.0', '8.1')
+
+ @property
+ def WindowsSdkLastVersion(self):
+ """
+ Microsoft Windows SDK last version
+ """
+ return self._use_last_dir_name(os.path.join(
+ self.WindowsSdkDir, 'lib'))
+
+ @property
+ def WindowsSdkDir(self):
+ """
+ Microsoft Windows SDK directory.
+ """
+ sdkdir = ''
+ for ver in self.WindowsSdkVersion:
+ # Try to get it from registry
+ loc = os.path.join(self.ri.windows_sdk, 'v%s' % ver)
+ sdkdir = self.ri.lookup(loc, 'installationfolder')
+ if sdkdir:
+ break
+ if not sdkdir or not os.path.isdir(sdkdir):
+ # Try to get "VC++ for Python" version from registry
+ path = os.path.join(self.ri.vc_for_python, '%0.1f' % self.vc_ver)
+ install_base = self.ri.lookup(path, 'installdir')
+ if install_base:
+ sdkdir = os.path.join(install_base, 'WinSDK')
+ if not sdkdir or not os.path.isdir(sdkdir):
+ # If fail, use default new path
+ for ver in self.WindowsSdkVersion:
+ intver = ver[:ver.rfind('.')]
+ path = r'Microsoft SDKs\Windows Kits\%s' % (intver)
+ d = os.path.join(self.ProgramFiles, path)
+ if os.path.isdir(d):
+ sdkdir = d
+ if not sdkdir or not os.path.isdir(sdkdir):
+ # If fail, use default old path
+ for ver in self.WindowsSdkVersion:
+ path = r'Microsoft SDKs\Windows\v%s' % ver
+ d = os.path.join(self.ProgramFiles, path)
+ if os.path.isdir(d):
+ sdkdir = d
+ if not sdkdir:
+ # If fail, use Platform SDK
+ sdkdir = os.path.join(self.VCInstallDir, 'PlatformSDK')
+ return sdkdir
+
+ @property
+ def WindowsSDKExecutablePath(self):
+ """
+ Microsoft Windows SDK executable directory.
+ """
+ # Find WinSDK NetFx Tools registry dir name
+ if self.vc_ver <= 11.0:
+ netfxver = 35
+ arch = ''
+ else:
+ netfxver = 40
+ hidex86 = True if self.vc_ver <= 12.0 else False
+ arch = self.pi.current_dir(x64=True, hidex86=hidex86)
+ fx = 'WinSDK-NetFx%dTools%s' % (netfxver, arch.replace('\\', '-'))
+
+ # liste all possibles registry paths
+ regpaths = []
+ if self.vc_ver >= 14.0:
+ for ver in self.NetFxSdkVersion:
+ regpaths += [os.path.join(self.ri.netfx_sdk, ver, fx)]
+
+ for ver in self.WindowsSdkVersion:
+ regpaths += [os.path.join(self.ri.windows_sdk, 'v%sA' % ver, fx)]
+
+ # Return installation folder from the more recent path
+ for path in regpaths:
+ execpath = self.ri.lookup(path, 'installationfolder')
+ if execpath:
+ break
+ return execpath
+
+ @property
+ def FSharpInstallDir(self):
+ """
+ Microsoft Visual F# directory.
+ """
+ path = r'%0.1f\Setup\F#' % self.vc_ver
+ path = os.path.join(self.ri.visualstudio, path)
+ return self.ri.lookup(path, 'productdir') or ''
+
+ @property
+ def UniversalCRTSdkDir(self):
+ """
+ Microsoft Universal CRT SDK directory.
+ """
+ # Set Kit Roots versions for specified MSVC++ version
+ if self.vc_ver >= 14.0:
+ vers = ('10', '81')
+ else:
+ vers = ()
+
+ # Find path of the more recent Kit
+ for ver in vers:
+ sdkdir = self.ri.lookup(self.ri.windows_kits_roots,
+ 'kitsroot%s' % ver)
+ if sdkdir:
+ break
+ return sdkdir or ''
+
+ @property
+ def UniversalCRTSdkLastVersion(self):
+ """
+ Microsoft Universal C Runtime SDK last version
+ """
+ return self._use_last_dir_name(os.path.join(
+ self.UniversalCRTSdkDir, 'lib'))
+
+ @property
+ def NetFxSdkVersion(self):
+ """
+ Microsoft .NET Framework SDK versions.
+ """
+ # Set FxSdk versions for specified MSVC++ version
+ if self.vc_ver >= 14.0:
+ return ('4.6.1', '4.6')
+ else:
+ return ()
+
+ @property
+ def NetFxSdkDir(self):
+ """
+ Microsoft .NET Framework SDK directory.
+ """
+ for ver in self.NetFxSdkVersion:
+ loc = os.path.join(self.ri.netfx_sdk, ver)
+ sdkdir = self.ri.lookup(loc, 'kitsinstallationfolder')
More information about the pypy-commit
mailing list