[pypy-commit] pypy py3.5: hg merge default
rlamy
pypy.commits at gmail.com
Thu May 25 21:08:26 EDT 2017
Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91412:a76798b64382
Date: 2017-05-26 02:08 +0100
http://bitbucket.org/pypy/pypy/changeset/a76798b64382/
Log: hg merge default
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -278,11 +278,14 @@
raise Exception("Cannot use the --output option with PyPy "
"when --shared is on (it is by default). "
"See issue #1971.")
- if (config.translation.profopt is not None
- and not config.translation.noprofopt):
- raise Exception("Cannot use the --profopt option "
- "when --shared is on (it is by default). "
- "See issue #2398.")
+
+ # if both profopt and profoptpath are specified then we keep them as they are with no other changes
+ if config.translation.profopt:
+ if config.translation.profoptargs is None:
+ config.translation.profoptargs = "$(RPYDIR)/../lib-python/2.7/test/regrtest.py --pgo -x test_asyncore test_gdb test_multiprocessing test_subprocess || true"
+ elif config.translation.profoptargs is not None:
+ raise Exception("Cannot use --profoptargs without specifying --profopt as well")
+
if sys.platform == 'win32':
libdir = thisdir.join('..', '..', 'libs')
libdir.ensure(dir=1)
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -830,6 +830,11 @@
executable = sys.pypy_find_executable(executable)
stdlib_path = sys.pypy_find_stdlib(executable)
if stdlib_path is None:
+ for lib_path in sys.path:
+ stdlib_path = sys.pypy_find_stdlib(lib_path)
+ if stdlib_path is not None:
+ break
+ if stdlib_path is None:
initstdio()
print(STDLIB_WARNING, file=sys.stderr)
else:
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -390,6 +390,9 @@
for i in range(len(string)):
self.ptr[start + i] = string[i]
+ def as_str(self):
+ return CBuffer(self).as_str()
+
def as_readbuf(self):
return CBuffer(self)
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -25,27 +25,27 @@
def test_index(self):
module = self.import_module(name='array')
- arr = module.array('i', [1,2,3,4])
+ arr = module.array('i', [1, 2, 3, 4])
assert arr[3] == 4
raises(IndexError, arr.__getitem__, 10)
del arr[2]
- assert arr.tolist() == [1,2,4]
+ assert arr.tolist() == [1, 2, 4]
arr[2] = 99
- assert arr.tolist() == [1,2,99]
+ assert arr.tolist() == [1, 2, 99]
def test_slice_get(self):
module = self.import_module(name='array')
- arr = module.array('i', [1,2,3,4])
- assert arr[:].tolist() == [1,2,3,4]
- assert arr[1:].tolist() == [2,3,4]
- assert arr[:2].tolist() == [1,2]
- assert arr[1:3].tolist() == [2,3]
+ arr = module.array('i', [1, 2, 3, 4])
+ assert arr[:].tolist() == [1, 2, 3, 4]
+ assert arr[1:].tolist() == [2, 3, 4]
+ assert arr[:2].tolist() == [1, 2]
+ assert arr[1:3].tolist() == [2, 3]
def test_slice_object(self):
module = self.import_module(name='array')
- arr = module.array('i', [1,2,3,4])
- assert arr[slice(1,3)].tolist() == [2,3]
- arr[slice(1,3)] = module.array('i', [21, 22, 23])
+ arr = module.array('i', [1, 2, 3, 4])
+ assert arr[slice(1, 3)].tolist() == [2,3]
+ arr[slice(1, 3)] = module.array('i', [21, 22, 23])
assert arr.tolist() == [1, 21, 22, 23, 4]
del arr[slice(1, 3)]
assert arr.tolist() == [1, 23, 4]
@@ -54,20 +54,15 @@
def test_buffer(self):
import sys
module = self.import_module(name='array')
- arr = module.array('i', [1,2,3,4])
+ arr = module.array('i', [1, 2, 3, 4])
buf = memoryview(arr)
exc = raises(TypeError, "buf[1] = 1")
assert str(exc.value) == "cannot modify read-only memory"
if sys.byteorder == 'big':
- assert bytes(buf) == (b'\0\0\0\x01'
- b'\0\0\0\x02'
- b'\0\0\0\x03'
- b'\0\0\0\x04')
+ expected = b'\0\0\0\x01\0\0\0\x02\0\0\0\x03\0\0\0\x04'
else:
- assert bytes(buf) == (b'\x01\0\0\0'
- b'\x02\0\0\0'
- b'\x03\0\0\0'
- b'\x04\0\0\0')
+ expected = b'\x01\0\0\0\x02\0\0\0\x03\0\0\0\x04\0\0\0'
+ assert bytes(buf) == expected
def test_releasebuffer(self):
module = self.import_module(name='array')
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -142,10 +142,9 @@
BoolOption("verbose", "Print extra information", default=False,
cmdline="--verbose"),
StrOption("cc", "Specify compiler to use for compiling generated C", cmdline="--cc"),
- StrOption("profopt", "Specify profile based optimization script",
- cmdline="--profopt"),
- BoolOption("noprofopt", "Don't use profile based optimization",
- default=False, cmdline="--no-profopt", negation=False),
+ BoolOption("profopt", "Enable profile guided optimization. Defaults to enabling this for PyPy. For other training workloads, please specify them in profoptargs",
+ cmdline="--profopt", default=False),
+ StrOption("profoptargs", "Absolute path to the profile guided optimization training script + the necessary arguments of the script", cmdline="--profoptargs", default=None),
BoolOption("instrument", "internal: turn instrumentation on",
default=False, cmdline=None),
BoolOption("countmallocs", "Count mallocs and frees", default=False,
diff --git a/rpython/jit/backend/arm/test/test_llop.py b/rpython/jit/backend/arm/test/test_llop.py
--- a/rpython/jit/backend/arm/test/test_llop.py
+++ b/rpython/jit/backend/arm/test/test_llop.py
@@ -5,5 +5,9 @@
class TestLLOp(JitARMMixin, _TestLLOp):
# for the individual tests see
# ====> ../../../metainterp/test/test_llop.py
- pass
+ # do NOT test the blackhole implementation of gc_store_indexed. It cannot
+ # work inside tests because llmodel.py:bh_gc_store_indexed_* receive a
+ # symbolic as the offset. It is not a problem because it is tested anyway
+ # by the same test in test_metainterp.py
+ TEST_BLACKHOLE = False
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -33,29 +33,6 @@
return python
-class ProfOpt(object):
- #XXX assuming gcc style flags for now
- name = "profopt"
-
- def __init__(self, compiler):
- self.compiler = compiler
-
- def first(self):
- return self.build('-fprofile-generate')
-
- def probe(self, exe, args):
- # 'args' is a single string typically containing spaces
- # and quotes, which represents several arguments.
- self.compiler.platform.execute(exe, args)
-
- def after(self):
- return self.build('-fprofile-use')
-
- def build(self, option):
- eci = ExternalCompilationInfo(compile_extra=[option],
- link_extra=[option])
- return self.compiler._build(eci)
-
class CCompilerDriver(object):
def __init__(self, platform, cfiles, eci, outputfilename=None,
profbased=False):
@@ -65,7 +42,7 @@
self.cfiles = cfiles
self.eci = eci
self.outputfilename = outputfilename
- self.profbased = profbased
+ # self.profbased = profbased
def _build(self, eci=ExternalCompilationInfo(), shared=False):
outputfilename = self.outputfilename
@@ -79,22 +56,6 @@
outputfilename=outputfilename,
standalone=not shared)
- def build(self, shared=False):
- if self.profbased:
- return self._do_profbased()
- return self._build(shared=shared)
-
- def _do_profbased(self):
- ProfDriver, args = self.profbased
- profdrv = ProfDriver(self)
- dolog = getattr(log, profdrv.name)
- dolog(args)
- exename = profdrv.first()
- dolog('Gathering profile data from: %s %s' % (
- str(exename), args))
- profdrv.probe(exename, args)
- return profdrv.after()
-
class CBuilder(object):
c_source_filename = None
_compiled = False
@@ -259,27 +220,6 @@
_entrypoint_wrapper = None
make_entrypoint_wrapper = True # for tests
- def getprofbased(self):
- profbased = None
- if self.config.translation.instrumentctl is not None:
- profbased = self.config.translation.instrumentctl
- else:
- # xxx handling config.translation.profopt is a bit messy, because
- # it could be an empty string (not to be confused with None) and
- # because noprofopt can be used as an override.
- profopt = self.config.translation.profopt
- if profopt is not None and not self.config.translation.noprofopt:
- profbased = (ProfOpt, profopt)
- return profbased
-
- def has_profopt(self):
- profbased = self.getprofbased()
- retval = (profbased and isinstance(profbased, tuple)
- and profbased[0] is ProfOpt)
- if retval and self.translator.platform.name == 'msvc':
- raise ValueError('Cannot do profile based optimization on MSVC,'
- 'it is not supported in free compiler version')
- return retval
def getentrypointptr(self):
# XXX check that the entrypoint has the correct
@@ -391,6 +331,8 @@
shared = self.config.translation.shared
extra_opts = []
+ if self.config.translation.profopt:
+ extra_opts += ["profopt"]
if self.config.translation.make_jobs != 1:
extra_opts += ['-j', str(self.config.translation.make_jobs)]
if self.config.translation.lldebug:
@@ -418,13 +360,12 @@
headers_to_precompile=headers_to_precompile,
no_precompile_cfiles = module_files,
shared=self.config.translation.shared,
+ profopt = self.config.translation.profopt,
config=self.config)
- if self.has_profopt():
- profopt = self.config.translation.profopt
- mk.definition('ABS_TARGET', str(targetdir.join('$(TARGET)')))
- mk.definition('DEFAULT_TARGET', 'profopt')
- mk.definition('PROFOPT', profopt)
+ if exe_name is None:
+ short = targetdir.basename
+ exe_name = targetdir.join(short)
rules = [
('debug', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT" debug_target'),
@@ -433,15 +374,28 @@
('llsafer', '', '$(MAKE) CFLAGS="-O2 -DRPY_LL_ASSERT" $(DEFAULT_TARGET)'),
('lldebug', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DRPY_LL_ASSERT" debug_target'),
('profile', '', '$(MAKE) CFLAGS="-g -O1 -pg $(CFLAGS) -fno-omit-frame-pointer" LDFLAGS="-pg $(LDFLAGS)" $(DEFAULT_TARGET)'),
- ]
- if self.has_profopt():
+ ]
+
+ # added a new target for profopt, because it requires -lgcov to compile successfully when -shared is used as an argument
+ # Also made a difference between translating with shared or not, because this affects profopt's target
+
+ if self.config.translation.profopt:
+ if self.config.translation.profoptargs is None:
+ raise Exception("No profoptargs specified, neither in the command line, nor in the target. If the target is not PyPy, please specify profoptargs")
+ if self.config.translation.shared:
+ mk.rule('$(PROFOPT_TARGET)', '$(TARGET) main.o',
+ '$(CC_LINK) $(LDFLAGS_LINK) main.o -L. -l$(SHARED_IMPORT_LIB) -o $@ $(RPATH_FLAGS) -lgcov')
+ else:
+ mk.definition('PROFOPT_TARGET', '$(TARGET)')
+
rules.append(
('profopt', '', [
- '$(MAKENOPROF)',
- '$(MAKE) CFLAGS="-fprofile-generate $(CFLAGS)" LDFLAGS="-fprofile-generate $(LDFLAGS)" $(TARGET)',
- 'cd $(RPYDIR)/translator/goal && $(ABS_TARGET) $(PROFOPT)',
- '$(MAKE) clean_noprof',
- '$(MAKE) CFLAGS="-fprofile-use $(CFLAGS)" LDFLAGS="-fprofile-use $(LDFLAGS)" $(TARGET)']))
+ '$(MAKE) CFLAGS="-fprofile-generate -fPIC $(CFLAGS) -fno-lto" LDFLAGS="-fprofile-generate $(LDFLAGS) -fno-lto" $(PROFOPT_TARGET)',
+ '%s %s ' % (exe_name, self.config.translation.profoptargs),
+ '$(MAKE) clean_noprof',
+ '$(MAKE) CFLAGS="-fprofile-use -fprofile-correction -fPIC $(CFLAGS) -fno-lto" LDFLAGS="-fprofile-use $(LDFLAGS) -fno-lto" $(PROFOPT_TARGET)',
+ ]))
+
for rule in rules:
mk.rule(*rule)
diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py
--- a/rpython/translator/c/test/test_standalone.py
+++ b/rpython/translator/c/test/test_standalone.py
@@ -241,20 +241,22 @@
os.write(1, str(tot))
return 0
from rpython.translator.interactive import Translation
- # XXX this is mostly a "does not crash option"
- t = Translation(entry_point, backend='c', profopt="100")
- # no counters
+ t = Translation(entry_point, backend='c', profopt=True, profoptargs="10", shared=True)
t.backendopt()
exe = t.compile()
- out = py.process.cmdexec("%s 500" % exe)
- assert int(out) == 500*501/2
- t = Translation(entry_point, backend='c', profopt="100",
- noprofopt=True)
- # no counters
+ assert (os.path.isfile("%s" % exe))
+
+ t = Translation(entry_point, backend='c', profopt=True, profoptargs="10", shared=False)
t.backendopt()
exe = t.compile()
- out = py.process.cmdexec("%s 500" % exe)
- assert int(out) == 500*501/2
+ assert (os.path.isfile("%s" % exe))
+
+ import rpython.translator.goal.targetrpystonedalone as rpy
+ t = Translation(rpy.entry_point, backend='c', profopt=True, profoptargs='1000', shared=False)
+ t.backendopt()
+ exe = t.compile()
+ assert (os.path.isfile("%s" % exe))
+
if hasattr(os, 'setpgrp'):
def test_os_setpgrp(self):
@@ -279,13 +281,12 @@
return 0
from rpython.translator.interactive import Translation
# XXX this is mostly a "does not crash option"
- t = Translation(entry_point, backend='c', profopt="")
+ t = Translation(entry_point, backend='c', profopt=True, profoptargs='10', shared=True)
# no counters
t.backendopt()
exe = t.compile()
#py.process.cmdexec(exe)
- t = Translation(entry_point, backend='c', profopt="",
- noprofopt=True)
+ t = Translation(entry_point, backend='c', profopt=True, profoptargs='10', shared=True)
# no counters
t.backendopt()
exe = t.compile()
diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py
--- a/rpython/translator/platform/posix.py
+++ b/rpython/translator/platform/posix.py
@@ -103,7 +103,7 @@
def gen_makefile(self, cfiles, eci, exe_name=None, path=None,
shared=False, headers_to_precompile=[],
- no_precompile_cfiles = [], config=None):
+ no_precompile_cfiles = [], profopt=False, config=None):
cfiles = self._all_cfiles(cfiles, eci)
if path is None:
@@ -189,6 +189,10 @@
('LINKFILES', eci.link_files),
('RPATH_FLAGS', self.get_rpath_flags(rel_libdirs)),
]
+
+ if profopt==True and shared==True:
+ definitions.append(('PROFOPT_TARGET', exe_name.basename))
+
for args in definitions:
m.definition(*args)
More information about the pypy-commit
mailing list