[pypy-svn] r9648 - pypy/dist/pypy/translator

[tismer at codespeak.net]

Author: tismer
Date: Fri Mar  4 20:15:55 2005
New Revision: 9648

Modified:
   pypy/dist/pypy/translator/genc.h
   pypy/dist/pypy/translator/genc.py
Log:
made the goal compile under windows, again.
First, the init code gets zlib compressed. This is
a nice code reduction, the source code gets below 12 MB.
But for windows, the size limit of string constants appears
to be some 2040, so I broke the string into 1K chunks
and modified the C code to collect these into a temp string.

Modified: pypy/dist/pypy/translator/genc.h
==============================================================================
--- pypy/dist/pypy/translator/genc.h	(original)
+++ pypy/dist/pypy/translator/genc.h	Fri Mar  4 20:15:55 2005
@@ -278,7 +278,7 @@
 	PyType_Ready(&PyGenCFunction_Type);	\
 	if (setup_globalfunctions(globalfunctiondefs) < 0) \
 		return;	\
-	if (setup_initcode(frozen_initcode, sizeof(frozen_initcode)) < 0) \
+	if (setup_initcode(frozen_initcode, FROZEN_INITCODE_SIZE) < 0) \
 		return;	\
 	if (setup_globalobjects(globalobjectdefs) < 0) \
 		return;
@@ -342,14 +342,29 @@
 	return 0;
 }
 
-static int setup_initcode(char* frozendata, int len)
+static int setup_initcode(char* frozendata[], int len)
 {
 	PyObject* co;
 	PyObject* globals;
 	PyObject* res;
-	co = PyMarshal_ReadObjectFromString(frozendata, len);
+	char *buffer, *bufp;
+	int chunk, count = 0;
+	
+	buffer = PyMem_NEW(char, len);
+	if (buffer == NULL)
+		return -1;
+	bufp = buffer;
+	while (count < len) {
+		chunk = len-count < 1024 ? len-count : 1024;
+		memcpy(bufp, *frozendata, chunk);
+		bufp += chunk;
+		count += chunk;
+		++frozendata;
+	}
+	co = PyMarshal_ReadObjectFromString(buffer, len);
 	if (co == NULL)
 		return -1;
+	PyMem_DEL(buffer);
 	if (!PyCode_Check(co)) {
 		PyErr_SetString(PyExc_TypeError, "uh?");
 		return -1;

Modified: pypy/dist/pypy/translator/genc.py
==============================================================================
--- pypy/dist/pypy/translator/genc.py	(original)
+++ pypy/dist/pypy/translator/genc.py	Fri Mar  4 20:15:55 2005
@@ -3,7 +3,7 @@
 
 """
 from __future__ import generators
-import autopath, os, sys, __builtin__, marshal
+import autopath, os, sys, __builtin__, marshal, zlib
 from pypy.objspace.flow.model import Variable, Constant, SpaceOperation
 from pypy.objspace.flow.model import FunctionGraph, Block, Link
 from pypy.objspace.flow.model import last_exception, last_exc_value
@@ -493,9 +493,12 @@
             if c in '\\"': return '\\' + c
             if ' ' <= c < '\x7F': return c
             return '\\%03o' % ord(c)
-        for i in range(0, len(bytecode), 20):
-            print >> f, ''.join([char_repr(c) for c in bytecode[i:i+20]])+'\\'
+        for i in range(0, len(bytecode), 32):
+            print >> f, ''.join([char_repr(c) for c in bytecode[i:i+32]])+'\\'
+            if (i+32) % 1024 == 0:
+                print >> f, self.C_FROZEN_BETWEEN
         print >> f, self.C_FROZEN_END
+        print >> f, "#define FROZEN_INITCODE_SIZE %d" % len(bytecode)
 
         # the footer proper: the module init function */
         print >> f, self.C_FOOTER % info
@@ -523,8 +526,14 @@
         del self.initcode[:]
         co = compile(source, self.modname, 'exec')
         del source
+        small = zlib.compress(marshal.dumps(co))
+        source = """if 1:
+            import zlib, marshal
+            exec marshal.loads(zlib.decompress(%r))""" % small
+        co = compile(source, self.modname, 'exec')
+        del source
         return marshal.dumps(co)
-
+    
     def gen_cfunction(self, func):
 ##         print 'gen_cfunction (%s:%d) %s' % (
 ##             func.func_globals.get('__name__', '?'),
@@ -831,9 +840,11 @@
 
     C_FROZEN_BEGIN = '''
 /* Frozen Python bytecode: the initialization code */
-static char frozen_initcode[] = "\\'''
+static char *frozen_initcode[] = {"\\'''
+
+    C_FROZEN_BETWEEN = '''", "\\'''
 
-    C_FROZEN_END = '''";\n'''
+    C_FROZEN_END = '''"};\n'''
 
     C_FOOTER = C_SEP + '''