[pypy-commit] pypy parallel-c-compilation: A first crude version of invoking multiple versions of a C compiler while
fijal
noreply at buildbot.pypy.org
Tue Apr 16 13:34:55 CEST 2013
Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: parallel-c-compilation
Changeset: r63405:2ac43e16ecda
Date: 2013-04-16 13:33 +0200
http://bitbucket.org/pypy/pypy/changeset/2ac43e16ecda/
Log: A first crude version of invoking multiple versions of a C compiler
while writing down source files
diff --git a/rpython/tool/runsubprocess2.py b/rpython/tool/runsubprocess2.py
new file mode 100644
--- /dev/null
+++ b/rpython/tool/runsubprocess2.py
@@ -0,0 +1,36 @@
+
+import os, sys, time, select
+from subprocess import Popen, PIPE
+import twisted
+
+_child = None
+_source = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+_source = os.path.join(os.path.join(_source, 'translator'), 'invoker.py')
+
+def spawn_subprocess():
+ global _child
+ _child = Popen([sys.executable, '-u', _source], bufsize=0,
+ stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True)
+spawn_subprocess()
+
+def cleanup_subprocess():
+ global _child
+ _child = None
+import atexit; atexit.register(cleanup_subprocess)
+
+def run(exe, *args):
+ _child.stdin.write(repr((exe,) + args) + "\n")
+
+class SubprocessExploded(Exception):
+ pass
+
+def results():
+ results = []
+ while True:
+ rl, _, _ = select.select([_child.stdout, _child.stderr], [], [], 0)
+ if _child.stderr in rl:
+ raise SubprocessExploded(_child.stderr)
+ elif _child.stdout in rl:
+ results.append(eval(_child.stdout.readline()))
+ else:
+ return results
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -1,6 +1,7 @@
import contextlib
+from rpython.tool import runsubprocess2
import py
-import sys, os
+import sys, os, time
from rpython.rlib import exports
from rpython.rtyper.typesystem import getfunctionptr
from rpython.tool import runsubprocess
@@ -112,7 +113,7 @@
str(exename), args))
profdrv.probe(exename, args)
return profdrv.after()
-
+
class CBuilder(object):
c_source_filename = None
_compiled = False
@@ -129,6 +130,7 @@
self.gcpolicy = gcpolicy # for tests only, e.g. rpython/memory/
self.eci = self.get_eci()
self.secondary_entrypoints = secondary_entrypoints
+ self._files_sent = 0
def get_eci(self):
pypy_include_dir = py.path.local(__file__).join('..')
@@ -149,7 +151,7 @@
thread_enabled=self.config.translation.thread,
sandbox=self.config.translation.sandbox)
self.db = db
-
+
# give the gc a chance to register interest in the start-up functions it
# need (we call this for its side-effects of db.get())
list(db.gcpolicy.gc_startup_code())
@@ -260,12 +262,40 @@
export_symbols=["pypy_main_startup"]))
self.eci, cfile, extra = gen_source(db, modulename, targetdir,
self.eci, defines=defines,
- split=self.split)
+ split=self.split,
+ invoke_c_callback=self._invoke_c_compiler)
self.c_source_filename = py.path.local(cfile)
self.extrafiles = self.eventually_copy(extra)
self.gen_makefile(targetdir, exe_name=exe_name)
return cfile
+ def _invoke_c_compiler(self, name, eci, directory):
+ if name is None or name.endswith('.h'):
+ return
+ shared = self.config.translation.shared
+ command = self.translator.platform.gen_cc_command(name, directory, eci,
+ shared=shared)
+ for name, rescode, out, err in runsubprocess2.results():
+ self._files_sent -= 1
+ if rescode == 0:
+ print "Compiled", name
+ else:
+ raise Exception("CC failed with", err)
+ self._files_sent += 1
+ runsubprocess2.run(name, *command)
+
+ def _wait_for_c_compiler(self):
+ while True:
+ for name, rescode, out, err in runsubprocess2.results():
+ if rescode == 0:
+ print "Compiled", name
+ else:
+ raise Exception("CC failed with", err)
+ self._files_sent -= 1
+ if self._files_sent == 0:
+ return
+ time.sleep(.1)
+
def eventually_copy(self, cfiles):
extrafiles = []
for fn in cfiles:
@@ -352,6 +382,7 @@
outputfilename=exe_name)
def compile(self, exe_name=None):
+ self._wait_for_c_compiler()
assert self.c_source_filename
assert not self._compiled
@@ -499,11 +530,14 @@
class SourceGenerator:
one_source_file = True
- def __init__(self, database):
+ def __init__(self, database, eci, invoke_c_callback=None):
self.database = database
+ self.eci = eci
self.extrafiles = []
self.path = None
self.namespace = NameManager()
+ self._last_file_name = None
+ self.invoke_c_callback = invoke_c_callback
def set_strategy(self, path, split=True):
all_nodes = list(self.database.globalcontainers())
@@ -526,6 +560,10 @@
return self.namespace.uniquename(name[:-2]) + '.c'
def makefile(self, name):
+ if self.invoke_c_callback is not None:
+ self.invoke_c_callback(self._last_file_name, self.path,
+ self.eci)
+ self._last_file_name = name
log.writing(name)
filepath = self.path.join(name)
if name.endswith('.c'):
@@ -621,7 +659,7 @@
if self.database.gcpolicy.need_no_typeptr():
pass # XXX gcc uses toooooons of memory???
else:
- split_criteria_big = SPLIT_CRITERIA * 4
+ split_criteria_big = SPLIT_CRITERIA * 4
#
# All declarations
@@ -776,7 +814,7 @@
def gen_source(database, modulename, targetdir,
- eci, defines={}, split=False):
+ eci, defines={}, split=False, invoke_c_callback=False):
if isinstance(targetdir, str):
targetdir = py.path.local(targetdir)
@@ -803,7 +841,7 @@
# 1) All declarations
# 2) Implementation of functions and global structures and arrays
#
- sg = SourceGenerator(database)
+ sg = SourceGenerator(database, eci, invoke_c_callback)
sg.set_strategy(targetdir, split)
database.prepare_inline_helpers()
sg.gen_readable_parts_of_source(f)
diff --git a/rpython/translator/invoker.py b/rpython/translator/invoker.py
new file mode 100644
--- /dev/null
+++ b/rpython/translator/invoker.py
@@ -0,0 +1,65 @@
+
+import os
+from twisted.internet import reactor
+from twisted.protocols import basic
+from twisted.internet import protocol, stdio
+from twisted.internet.error import ProcessDone
+
+class SubProcessProtocol(protocol.ProcessProtocol):
+ def __init__(self, commander, name):
+ self._commander = commander
+ self._output = []
+ self._name = name
+ self._error = []
+
+ def outReceived(self, out):
+ self._output.append(out)
+
+ def errReceived(self, err):
+ self._error.append(err)
+
+ def processExited(self, status):
+ self._commander.process_exited()
+ if status.value == ProcessDone:
+ print repr((self._name, 0, "".join(self._output),
+ "".join(self._error)))
+ else:
+ print repr((self._name, status.value.exitCode,
+ "".join(self._output), "".join(self._error)))
+
+MAX = 8
+
+class Commander(basic.LineReceiver):
+ delimiter = '\n'
+
+ def __init__(self, max=MAX):
+ self._counter = 0
+ self._max = max
+ self._queue = []
+
+ def connectionMade(self):
+ pass
+
+ def process_exited(self):
+ self._counter -= 1
+ if self._queue:
+ self._run(self._queue.pop())
+
+ def _run(self, line):
+ args = eval(line)
+ reactor.spawnProcess(SubProcessProtocol(self, args[0]), args[1],
+ args[1:], env=os.environ.copy())
+ self._counter += 1
+
+ def lineReceived(self, line):
+ if not line:
+ reactor.stop()
+ return
+ assert line.startswith('(')
+ if self._counter < self._max:
+ self._run(line)
+ else:
+ self._queue.append(line)
+
+stdio.StandardIO(Commander())
+reactor.run()
diff --git a/rpython/translator/platform/__init__.py b/rpython/translator/platform/__init__.py
--- a/rpython/translator/platform/__init__.py
+++ b/rpython/translator/platform/__init__.py
@@ -246,6 +246,9 @@
def _library_dirs_for_libffi(self):
raise NotImplementedError("Needs to be overwritten")
+ def gen_cc_command(self, fname, eci, shared=False):
+ raise NotImplementedError("abstract base class")
+
def check___thread(self):
return True
diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py
--- a/rpython/translator/platform/posix.py
+++ b/rpython/translator/platform/posix.py
@@ -85,6 +85,12 @@
# strip compiler flags
return [entry[2:] for entry in out.split()]
+ def gen_cc_command(self, fname, eci, path, shared=False):
+ fullname = str(path.join(fname))
+ objfullname = fullname[:-2] + '.o'
+ return ([self.cc] + self._compile_args_from_eci(eci, not shared) +
+ ['-c', '-o', objfullname, fullname])
+
def gen_makefile(self, cfiles, eci, exe_name=None, path=None,
shared=False):
cfiles = self._all_cfiles(cfiles, eci)
@@ -250,7 +256,7 @@
self.defs = {}
self.lines = []
self.makefile_dir = py.path.local(path)
-
+
def pathrel(self, fpath):
if fpath.dirpath() == self.makefile_dir:
return fpath.basename
More information about the pypy-commit
mailing list