[pypy-commit] pypy default: Merge branch 'llimpl'
rlamy
pypy.commits at gmail.com
Wed Feb 17 17:59:58 EST 2016
Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch:
Changeset: r82316:4c30448f0457
Date: 2016-02-17 22:59 +0000
http://bitbucket.org/pypy/pypy/changeset/4c30448f0457/
Log: Merge branch 'llimpl'
Refactor register_external(), remove running_on_llinterp mechanism
and apply sandbox transform on externals at the end of annotation.
diff --git a/rpython/annotator/policy.py b/rpython/annotator/policy.py
--- a/rpython/annotator/policy.py
+++ b/rpython/annotator/policy.py
@@ -3,6 +3,9 @@
from rpython.annotator.specialize import (
specialize_argvalue, specialize_argtype, specialize_arglistitemtype,
specialize_arg_or_var, memo, specialize_call_location)
+from rpython.flowspace.operation import op
+from rpython.flowspace.model import Constant
+from rpython.annotator.model import SomeTuple
class AnnotatorPolicy(object):
@@ -64,7 +67,34 @@
return LowLevelAnnotatorPolicy.specialize__ll_and_arg(*args)
def no_more_blocks_to_annotate(pol, annotator):
+ bk = annotator.bookkeeper
# hint to all pending specializers that we are done
- for callback in annotator.bookkeeper.pending_specializations:
+ for callback in bk.pending_specializations:
callback()
- del annotator.bookkeeper.pending_specializations[:]
+ del bk.pending_specializations[:]
+ if annotator.added_blocks is not None:
+ all_blocks = annotator.added_blocks
+ else:
+ all_blocks = annotator.annotated
+ for block in list(all_blocks):
+ for i, instr in enumerate(block.operations):
+ if not isinstance(instr, (op.simple_call, op.call_args)):
+ continue
+ v_func = instr.args[0]
+ s_func = annotator.annotation(v_func)
+ if not hasattr(s_func, 'needs_sandboxing'):
+ continue
+ key = ('sandboxing', s_func.const)
+ if key not in bk.emulated_pbc_calls:
+ params_s = s_func.args_s
+ s_result = s_func.s_result
+ from rpython.translator.sandbox.rsandbox import make_sandbox_trampoline
+ sandbox_trampoline = make_sandbox_trampoline(
+ s_func.name, params_s, s_result)
+ sandbox_trampoline._signature_ = [SomeTuple(items=params_s)], s_result
+ bk.emulate_pbc_call(key, bk.immutablevalue(sandbox_trampoline), params_s)
+ else:
+ s_trampoline = bk.emulated_pbc_calls[key][0]
+ sandbox_trampoline = s_trampoline.const
+ new = instr.replace({instr.args[0]: Constant(sandbox_trampoline)})
+ block.operations[i] = new
diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -113,8 +113,9 @@
@op.simple_call.register(SomeObject)
def simple_call_SomeObject(annotator, func, *args):
- return annotator.annotation(func).call(
- simple_args([annotator.annotation(arg) for arg in args]))
+ s_func = annotator.annotation(func)
+ argspec = simple_args([annotator.annotation(arg) for arg in args])
+ return s_func.call(argspec)
@op.call_args.register_transform(SomeObject)
def transform_varargs(annotator, v_func, v_shape, *data_v):
diff --git a/rpython/memory/gctransform/test/test_transform.py b/rpython/memory/gctransform/test/test_transform.py
--- a/rpython/memory/gctransform/test/test_transform.py
+++ b/rpython/memory/gctransform/test/test_transform.py
@@ -5,6 +5,7 @@
from rpython.translator.exceptiontransform import ExceptionTransformer
from rpython.rtyper.lltypesystem import lltype
from rpython.conftest import option
+from rpython.rtyper.rtyper import llinterp_backend
class LLInterpedTranformerTests:
@@ -131,8 +132,10 @@
def rtype(func, inputtypes, specialize=True):
t = TranslationContext()
t.buildannotator().build_types(func, inputtypes)
+ rtyper = t.buildrtyper()
+ rtyper.backend = llinterp_backend
if specialize:
- t.buildrtyper().specialize()
+ rtyper.specialize()
if option.view:
t.view()
return t
diff --git a/rpython/memory/test/test_transformed_gc.py b/rpython/memory/test/test_transformed_gc.py
--- a/rpython/memory/test/test_transformed_gc.py
+++ b/rpython/memory/test/test_transformed_gc.py
@@ -14,6 +14,7 @@
from rpython.conftest import option
from rpython.rlib.rstring import StringBuilder
from rpython.rlib.rarithmetic import LONG_BIT
+from rpython.rtyper.rtyper import llinterp_backend
WORD = LONG_BIT // 8
@@ -29,9 +30,11 @@
t.config.set(**extraconfigopts)
ann = t.buildannotator()
ann.build_types(func, inputtypes)
+ rtyper = t.buildrtyper()
+ rtyper.backend = llinterp_backend
if specialize:
- t.buildrtyper().specialize()
+ rtyper.specialize()
if backendopt:
from rpython.translator.backendopt.all import backend_optimizations
backend_optimizations(t)
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -275,8 +275,6 @@
return lltype.Signed
malloc_zero_filled = CDefinedIntSymbolic('MALLOC_ZERO_FILLED', default=0)
-running_on_llinterp = CDefinedIntSymbolic('RUNNING_ON_LLINTERP', default=1)
-# running_on_llinterp is meant to have the value 0 in all backends
# ____________________________________________________________
diff --git a/rpython/rtyper/extfunc.py b/rpython/rtyper/extfunc.py
--- a/rpython/rtyper/extfunc.py
+++ b/rpython/rtyper/extfunc.py
@@ -1,99 +1,105 @@
-from rpython.rtyper.extregistry import ExtRegistryEntry
-from rpython.rtyper.lltypesystem.lltype import typeOf, FuncType, functionptr
-from rpython.annotator.model import unionof
+from rpython.annotator.model import unionof, SomeObject
from rpython.annotator.signature import annotation, SignatureError
+from rpython.rtyper.extregistry import ExtRegistryEntry, lookup
+from rpython.rtyper.lltypesystem.lltype import (
+ typeOf, FuncType, functionptr, _ptr, Void)
+from rpython.rtyper.error import TyperError
+from rpython.rtyper.rmodel import Repr
-import py
+class SomeExternalFunction(SomeObject):
+ def __init__(self, name, args_s, s_result):
+ self.name = name
+ self.args_s = args_s
+ self.s_result = s_result
+
+ def check_args(self, callspec):
+ params_s = self.args_s
+ args_s, kwargs = callspec.unpack()
+ if kwargs:
+ raise SignatureError(
+ "External functions cannot be called with keyword arguments")
+ if len(args_s) != len(params_s):
+ raise SignatureError("Argument number mismatch")
+ for i, s_param in enumerate(params_s):
+ arg = unionof(args_s[i], s_param)
+ if not s_param.contains(arg):
+ raise SignatureError(
+ "In call to external function %r:\n"
+ "arg %d must be %s,\n"
+ " got %s" % (
+ self.name, i + 1, s_param, args_s[i]))
+
+ def call(self, callspec):
+ self.check_args(callspec)
+ return self.s_result
+
+ def rtyper_makerepr(self, rtyper):
+ if not self.is_constant():
+ raise TyperError("Non-constant external function!")
+ entry = lookup(self.const)
+ impl = getattr(entry, 'lltypeimpl', None)
+ fakeimpl = getattr(entry, 'lltypefakeimpl', None)
+ return ExternalFunctionRepr(self, impl, fakeimpl)
+
+ def rtyper_makekey(self):
+ return self.__class__, self
+
+class ExternalFunctionRepr(Repr):
+ lowleveltype = Void
+
+ def __init__(self, s_func, impl, fakeimpl):
+ self.s_func = s_func
+ self.impl = impl
+ self.fakeimpl = fakeimpl
+
+ def rtype_simple_call(self, hop):
+ rtyper = hop.rtyper
+ args_r = [rtyper.getrepr(s_arg) for s_arg in self.s_func.args_s]
+ r_result = rtyper.getrepr(self.s_func.s_result)
+ obj = self.get_funcptr(rtyper, args_r, r_result)
+ hop2 = hop.copy()
+ hop2.r_s_popfirstarg()
+ vlist = [hop2.inputconst(typeOf(obj), obj)] + hop2.inputargs(*args_r)
+ hop2.exception_is_here()
+ return hop2.genop('direct_call', vlist, r_result)
+
+ def get_funcptr(self, rtyper, args_r, r_result):
+ from rpython.rtyper.rtyper import llinterp_backend
+ args_ll = [r_arg.lowleveltype for r_arg in args_r]
+ ll_result = r_result.lowleveltype
+ name = self.s_func.name
+ if self.fakeimpl and rtyper.backend is llinterp_backend:
+ FT = FuncType(args_ll, ll_result)
+ return functionptr(
+ FT, name, _external_name=name, _callable=self.fakeimpl)
+ elif self.impl:
+ if isinstance(self.impl, _ptr):
+ return self.impl
+ else:
+ # store some attributes to the 'impl' function, where
+ # the eventual call to rtyper.getcallable() will find them
+ # and transfer them to the final lltype.functionptr().
+ self.impl._llfnobjattrs_ = {'_name': name}
+ return rtyper.getannmixlevel().delayedfunction(
+ self.impl, self.s_func.args_s, self.s_func.s_result)
+ else:
+ fakeimpl = self.fakeimpl or self.s_func.const
+ FT = FuncType(args_ll, ll_result)
+ return functionptr(
+ FT, name, _external_name=name, _callable=fakeimpl)
+
class ExtFuncEntry(ExtRegistryEntry):
safe_not_sandboxed = False
- # common case: args is a list of annotation or types
- def normalize_args(self, *args_s):
- args = self.signature_args
- signature_args = [annotation(arg, None) for arg in args]
- assert len(args_s) == len(signature_args),\
- "Argument number mismatch"
+ def compute_annotation(self):
+ s_result = SomeExternalFunction(
+ self.name, self.signature_args, self.signature_result)
+ if (self.bookkeeper.annotator.translator.config.translation.sandbox
+ and not self.safe_not_sandboxed):
+ s_result.needs_sandboxing = True
+ return s_result
- for i, expected in enumerate(signature_args):
- arg = unionof(args_s[i], expected)
- if not expected.contains(arg):
- name = getattr(self, 'name', None)
- if not name:
- try:
- name = self.instance.__name__
- except AttributeError:
- name = '?'
- raise SignatureError("In call to external function %r:\n"
- "arg %d must be %s,\n"
- " got %s" % (
- name, i+1, expected, args_s[i]))
- return signature_args
-
- def compute_result_annotation(self, *args_s):
- self.normalize_args(*args_s) # check arguments
- return self.signature_result
-
- def specialize_call(self, hop):
- rtyper = hop.rtyper
- signature_args = self.normalize_args(*hop.args_s)
- args_r = [rtyper.getrepr(s_arg) for s_arg in signature_args]
- args_ll = [r_arg.lowleveltype for r_arg in args_r]
- s_result = hop.s_result
- r_result = rtyper.getrepr(s_result)
- ll_result = r_result.lowleveltype
- name = getattr(self, 'name', None) or self.instance.__name__
- impl = getattr(self, 'lltypeimpl', None)
- fakeimpl = getattr(self, 'lltypefakeimpl', self.instance)
- if impl:
- if (rtyper.annotator.translator.config.translation.sandbox
- and not self.safe_not_sandboxed):
- from rpython.translator.sandbox.rsandbox import (
- make_sandbox_trampoline)
- impl = make_sandbox_trampoline(
- self.name, signature_args, s_result)
- if hasattr(self, 'lltypefakeimpl'):
- # If we have both an llimpl and an llfakeimpl,
- # we need a wrapper that selects the proper one and calls it
- from rpython.tool.sourcetools import func_with_new_name
- # Using '*args' is delicate because this wrapper is also
- # created for init-time functions like llarena.arena_malloc
- # which are called before the GC is fully initialized
- args = ', '.join(['arg%d' % i for i in range(len(args_ll))])
- d = {'original_impl': impl,
- 's_result': s_result,
- 'fakeimpl': fakeimpl,
- '__name__': __name__,
- }
- exec py.code.compile("""
- from rpython.rlib.objectmodel import running_on_llinterp
- from rpython.rlib.debug import llinterpcall
- from rpython.rlib.jit import dont_look_inside
- # note: we say 'dont_look_inside' mostly because the
- # JIT does not support 'running_on_llinterp', but in
- # theory it is probably right to stop jitting anyway.
- @dont_look_inside
- def ll_wrapper(%s):
- if running_on_llinterp:
- return llinterpcall(s_result, fakeimpl, %s)
- else:
- return original_impl(%s)
- """ % (args, args, args)) in d
- impl = func_with_new_name(d['ll_wrapper'], name + '_wrapper')
- # store some attributes to the 'impl' function, where
- # the eventual call to rtyper.getcallable() will find them
- # and transfer them to the final lltype.functionptr().
- impl._llfnobjattrs_ = {'_name': self.name}
- obj = rtyper.getannmixlevel().delayedfunction(
- impl, signature_args, hop.s_result)
- else:
- FT = FuncType(args_ll, ll_result)
- obj = functionptr(FT, name, _external_name=self.name,
- _callable=fakeimpl,
- _safe_not_sandboxed=self.safe_not_sandboxed)
- vlist = [hop.inputconst(typeOf(obj), obj)] + hop.inputargs(*args_r)
- hop.exception_is_here()
- return hop.genop('direct_call', vlist, r_result)
def register_external(function, args, result=None, export_name=None,
llimpl=None, llfakeimpl=None, sandboxsafe=False):
@@ -109,32 +115,20 @@
if export_name is None:
export_name = function.__name__
+ params_s = [annotation(arg) for arg in args]
+ s_result = annotation(result)
class FunEntry(ExtFuncEntry):
_about_ = function
safe_not_sandboxed = sandboxsafe
-
- if args is None:
- def normalize_args(self, *args_s):
- return args_s # accept any argument unmodified
- elif callable(args):
- # custom annotation normalizer (see e.g. os.utime())
- normalize_args = staticmethod(args)
- else: # use common case behavior
- signature_args = args
-
- signature_result = annotation(result, None)
+ signature_args = params_s
+ signature_result = s_result
name = export_name
if llimpl:
lltypeimpl = staticmethod(llimpl)
if llfakeimpl:
lltypefakeimpl = staticmethod(llfakeimpl)
- if export_name:
- FunEntry.__name__ = export_name
- else:
- FunEntry.__name__ = function.func_name
-
def is_external(func):
if hasattr(func, 'value'):
func = func.value
diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py
--- a/rpython/rtyper/rtyper.py
+++ b/rpython/rtyper/rtyper.py
@@ -32,11 +32,24 @@
from rpython.translator.sandbox.rsandbox import make_sandbox_trampoline
+class RTyperBackend(object):
+ pass
+
+class GenCBackend(RTyperBackend):
+ pass
+genc_backend = GenCBackend()
+
+class LLInterpBackend(RTyperBackend):
+ pass
+llinterp_backend = LLInterpBackend()
+
+
class RPythonTyper(object):
from rpython.rtyper.rmodel import log
- def __init__(self, annotator):
+ def __init__(self, annotator, backend=genc_backend):
self.annotator = annotator
+ self.backend = backend
self.lowlevel_ann_policy = LowLevelAnnotatorPolicy(self)
self.reprs = {}
self._reprs_must_call_setup = []
diff --git a/rpython/rtyper/test/test_extfunc.py b/rpython/rtyper/test/test_extfunc.py
--- a/rpython/rtyper/test/test_extfunc.py
+++ b/rpython/rtyper/test/test_extfunc.py
@@ -1,7 +1,6 @@
import py
-from rpython.rtyper.extfunc import ExtFuncEntry, register_external,\
- is_external
+from rpython.rtyper.extfunc import register_external
from rpython.annotator.model import SomeInteger, SomeString, AnnotatorError
from rpython.annotator.annrpython import RPythonAnnotator
from rpython.annotator.policy import AnnotatorPolicy
@@ -19,11 +18,7 @@
"NOT_RPYTHON"
return eval("x+40")
- class BTestFuncEntry(ExtFuncEntry):
- _about_ = b
- name = 'b'
- signature_args = [SomeInteger()]
- signature_result = SomeInteger()
+ register_external(b, [int], result=int)
def f():
return b(2)
@@ -43,15 +38,11 @@
def c(y, x):
yyy
- class CTestFuncEntry(ExtFuncEntry):
- _about_ = c
- name = 'ccc'
- signature_args = [SomeInteger()] * 2
- signature_result = SomeInteger()
+ def llimpl(y, x):
+ return y + x
- def lltypeimpl(y, x):
- return y + x
- lltypeimpl = staticmethod(lltypeimpl)
+ register_external(c, [int, int], result=int, llimpl=llimpl,
+ export_name='ccc')
def f():
return c(3, 4)
@@ -59,22 +50,6 @@
res = interpret(f, [])
assert res == 7
- def test_register_external_signature(self):
- """
- Test the standard interface for external functions.
- """
- def dd():
- pass
- register_external(dd, [int], int)
-
- def f():
- return dd(3)
-
- policy = AnnotatorPolicy()
- a = RPythonAnnotator(policy=policy)
- s = a.build_types(f, [])
- assert isinstance(s, SomeInteger)
-
def test_register_external_tuple_args(self):
"""
Verify the annotation of a registered external function which takes a
@@ -121,23 +96,6 @@
s = a.build_types(f, [])
assert isinstance(s, SomeInteger)
- def test_register_external_specialcase(self):
- """
- When args=None, the external function accepts any arguments unmodified.
- """
- def function_withspecialcase(arg):
- return repr(arg)
- register_external(function_withspecialcase, args=None, result=str)
-
- def f():
- x = function_withspecialcase
- return x(33) + x("aaa") + x([]) + "\n"
-
- policy = AnnotatorPolicy()
- a = RPythonAnnotator(policy=policy)
- s = a.build_types(f, [])
- assert isinstance(s, SomeString)
-
def test_str0(self):
str0 = SomeString(no_nul=True)
def os_open(s):
@@ -182,3 +140,22 @@
# fails with TooLateForChange
a.build_types(g, [[str]])
a.build_types(g, [[str0]]) # Does not raise
+
+ def test_register_external_llfakeimpl(self):
+ def a(i):
+ return i
+ def a_llimpl(i):
+ return i * 2
+ def a_llfakeimpl(i):
+ return i * 3
+ register_external(a, [int], int, llimpl=a_llimpl,
+ llfakeimpl=a_llfakeimpl)
+ def f(i):
+ return a(i)
+
+ res = interpret(f, [7])
+ assert res == 21
+
+ from rpython.translator.c.test.test_genc import compile
+ fc = compile(f, [int])
+ assert fc(7) == 14
diff --git a/rpython/rtyper/test/test_llinterp.py b/rpython/rtyper/test/test_llinterp.py
--- a/rpython/rtyper/test/test_llinterp.py
+++ b/rpython/rtyper/test/test_llinterp.py
@@ -13,7 +13,7 @@
from rpython.rlib.rarithmetic import r_uint, ovfcheck
from rpython.tool import leakfinder
from rpython.conftest import option
-
+from rpython.rtyper.rtyper import llinterp_backend
# switch on logging of interp to show more info on failing tests
@@ -39,6 +39,7 @@
t.view()
global typer # we need it for find_exception
typer = t.buildrtyper()
+ typer.backend = llinterp_backend
typer.specialize()
#t.view()
t.checkgraphs()
diff --git a/rpython/rtyper/test/test_rbuiltin.py b/rpython/rtyper/test/test_rbuiltin.py
--- a/rpython/rtyper/test/test_rbuiltin.py
+++ b/rpython/rtyper/test/test_rbuiltin.py
@@ -3,8 +3,7 @@
import py
-from rpython.rlib.debug import llinterpcall
-from rpython.rlib.objectmodel import instantiate, running_on_llinterp, compute_unique_id, current_object_addr_as_int
+from rpython.rlib.objectmodel import instantiate, compute_unique_id, current_object_addr_as_int
from rpython.rlib.rarithmetic import (intmask, longlongmask, r_int64, is_valid_int,
r_int, r_uint, r_longlong, r_ulonglong)
from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
@@ -456,26 +455,6 @@
res = self.interpret(fn, [3.25])
assert res == 7.25
- def test_debug_llinterpcall(self):
- S = lltype.Struct('S', ('m', lltype.Signed))
- SPTR = lltype.Ptr(S)
- def foo(n):
- "NOT_RPYTHON"
- s = lltype.malloc(S, immortal=True)
- s.m = eval("n*6", locals())
- return s
- def fn(n):
- if running_on_llinterp:
- return llinterpcall(SPTR, foo, n).m
- else:
- return 321
- res = self.interpret(fn, [7])
- assert res == 42
- from rpython.translator.c.test.test_genc import compile
- f = compile(fn, [int])
- res = f(7)
- assert res == 321
-
def test_id(self):
class A:
pass
diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py
--- a/rpython/translator/c/node.py
+++ b/rpython/translator/c/node.py
@@ -913,6 +913,7 @@
return []
def new_funcnode(db, T, obj, forcename=None):
+ from rpython.rtyper.rtyper import llinterp_backend
if db.sandbox:
if (getattr(obj, 'external', None) is not None and
not obj._safe_not_sandboxed):
@@ -934,6 +935,9 @@
return ExternalFuncNode(db, T, obj, name)
elif hasattr(obj._callable, "c_name"):
return ExternalFuncNode(db, T, obj, name) # this case should only be used for entrypoints
+ elif db.translator.rtyper.backend is llinterp_backend:
+ # on llinterp, anything goes
+ return ExternalFuncNode(db, T, obj, name)
else:
raise ValueError("don't know how to generate code for %r" % (obj,))
diff --git a/rpython/translator/sandbox/test/test_sandbox.py b/rpython/translator/sandbox/test/test_sandbox.py
--- a/rpython/translator/sandbox/test/test_sandbox.py
+++ b/rpython/translator/sandbox/test/test_sandbox.py
@@ -292,6 +292,21 @@
rescode = pipe.wait()
assert rescode == 0
+def test_environ_items():
+ def entry_point(argv):
+ print os.environ.items()
+ return 0
+
+ exe = compile(entry_point)
+ g, f = run_in_subprocess(exe)
+ expect(f, g, "ll_os.ll_os_envitems", (), [])
+ expect(f, g, "ll_os.ll_os_write", (1, "[]\n"), 3)
+ g.close()
+ tail = f.read()
+ f.close()
+ assert tail == ""
+
+
class TestPrintedResults:
def run(self, entry_point, args, expected):
More information about the pypy-commit
mailing list