[pypy-svn] r13502 - pypy/dist/pypy/translator/pickle
tismer at codespeak.net
tismer at codespeak.net
Fri Jun 17 01:47:37 CEST 2005
Author: tismer
Date: Fri Jun 17 01:47:36 2005
New Revision: 13502
Modified:
pypy/dist/pypy/translator/pickle/genpickle.py
pypy/dist/pypy/translator/pickle/loader.py
pypy/dist/pypy/translator/pickle/main.py
pypy/dist/pypy/translator/pickle/writer.py
Log:
quite a rewrite of genpickle.
This version does not have any latercode, anymore.
There is a need to increase the recursionlimit, but only marginal.
The key point is to register objects as soon as possible.
This check-in was overdue, the interp thing got into the way.
Modified: pypy/dist/pypy/translator/pickle/genpickle.py
==============================================================================
--- pypy/dist/pypy/translator/pickle/genpickle.py (original)
+++ pypy/dist/pypy/translator/pickle/genpickle.py Fri Jun 17 01:47:36 2005
@@ -33,6 +33,8 @@
from types import *
import types
+class AlreadyCreated(Exception): pass
+
# ____________________________________________________________
@@ -40,188 +42,246 @@
def __init__(self, translator, writer = None):
self.translator = translator
- self.initcode = [
+ self.writer = writer
+ self.initcode = []
+ self.produce = self._produce()
+ self.produce(
'from __future__ import %s\n' % ', '.join(all_feature_names) +
'import new, types, sys',
- ]
-
- self.latercode = [] # list of generators generating extra lines
- self.debugstack = () # linked list of nested nameof()
-
- self.picklenames = {Constant(None): 'None',
- Constant(False): 'False',
- Constant(True): 'True',
- # hack: overflowed float
- Constant(float("1e10000000000000000000000000000000")):
- 'float("1e10000000000000000000000000000000")',
- }
+ )
+ self.picklenames = {} # memoize objects
+ self.memoize(float("1e10000000000000000000000000000000"),
+ 'float("1e10000000000000000000000000000000")')
for name in all_feature_names + "new types sys".split():
- self.picklenames[Constant(globals()[name])] = name
+ self.memoize(globals()[name], name)
self.namespace = NameManager()
+ self.uniquename = self.namespace.uniquename
self.namespace.make_reserved_names('None False True')
self.namespace.make_reserved_names('new types sys')
self.namespace.make_reserved_names(' '.join(all_feature_names))
+ self.namespace.make_reserved_names('result') # result dict
+ self.result = {}
+ self.simple_const_types = {
+ int: repr,
+ long: repr,
+ float: repr,
+ str: repr,
+ unicode: repr,
+ type(None): repr,
+ bool: repr,
+ }
+ self.typecache = {} # hold types vs. nameof methods
# we distinguish between the "user program" and other stuff.
# "user program" will never use save_global.
self.domains = (
'pypy.objspace.std.',
+ 'pypy.objspace.descroperation',
'pypy._cache.',
'pypy.interpreter.',
'pypy.module.',
'__main__',
)
self.shortnames = {
- SpaceOperation: 'SOP',
Variable: 'V',
Constant: 'C',
Block: 'B',
SpamBlock: 'SB',
EggBlock: 'EB',
Link: 'L',
- FunctionGraph: 'F',
- SomeInteger: 'SI',
- SomeObject: 'SO',
- SomeChar: 'SC',
- SomeBool: 'SB',
- SomeList: 'SL',
- SomeString: 'SS',
- SomeTuple: 'ST',
- SomeInstance: 'SIN',
+ FunctionGraph: 'FG',
+ SomeInteger: 'sI',
+ SomeObject: 'sO',
+ SomeChar: 'sC',
+ SomeBool: 'sB',
+ SomeList: 'sL',
+ SomeString: 'sS',
+ SomeTuple: 'sT',
+ SomeInstance: 'sIn',
}
- self.writer = writer
+ self.inline_instances = {
+ SpaceOperation: True,
+ }
+
+ def pickle(self, **kwds):
+ for obj in kwds.values():
+ self.nameof(obj)
+ self.result.update(kwds)
+
+ def finish(self):
+ self.nameof(self.result)
+ self.pickle()
+ self.produce('result = %s' % self.nameof(self.result))
+ if self.writer:
+ self.writer.close()
+
+ def memoize(self, obj, name):
+ self.picklenames[id(obj)] = name
+ return name
+
+ def memoize_unique(self, obj, basename):
+ if id(obj) in self.picklenames:
+ raise AlreadyCreated
+ return self.memoize(obj, self.uniquename(basename))
- def nameof(self, obj, debug=None, namehint=None):
- key = Constant(obj)
+ def _produce(self):
+ writer = self.writer
+ down = 1234
+ cnt = [0, 0] # text, calls
+ self.last_progress = ''
+ if writer:
+ write = writer.write
+ else:
+ write = self.initcode.append
+ def produce(text):
+ write(text+'\n')
+ cnt[0] += len(text) + 1
+ cnt[1] += 1
+ if cnt[1] == down:
+ cnt[1] = 0
+ self.progress("%d" % cnt[0])
+ return produce
+
+ def progress(self, txt):
+ back = '\x08' * len(self.last_progress)
+ self.last_progress = txt+' ' # soft space
+ print back+txt,
+
+ def spill(self):
+ self.progress_count += len(self.initcode)
+ writer = self.writer
+ if writer:
+ for line in self.initcode:
+ writer.write(line+'\n')
+ del self.initcode[:]
+ if self.progress_count - self.progress_last >= 1234:
+ print '%s%d' % (20*'\x08', self.progress_count),
+ self.progress_last = self.progress_count
+
+ def nameof(self, obj):
try:
- return self.picklenames[key]
+ try:
+ return self.picklenames[id(obj)]
+ except KeyError:
+ typ = type(obj)
+ return self.simple_const_types[typ](obj)
except KeyError:
- if debug:
- stackentry = debug, obj
- else:
- stackentry = obj
- self.debugstack = (self.debugstack, stackentry)
- obj_builtin_base = builtin_base(obj)
- if obj_builtin_base in (object, int, long) and type(obj) is not obj_builtin_base:
- # assume it's a user defined thingy
- name = self.nameof_instance(obj)
- else:
- for cls in type(obj).__mro__:
- meth = getattr(self,
- 'nameof_' + ''.join( [
- c for c in cls.__name__
- if c.isalpha() or c == '_'] ),
- None)
- if meth:
- break
- else:
- raise Exception, "nameof(%r)" % (obj,)
-
- code = meth.im_func.func_code
- if namehint and 'namehint' in code.co_varnames[:code.co_argcount]:
- name = meth(obj, namehint=namehint)
- else:
- name = meth(obj)
- self.debugstack, x = self.debugstack
- assert x is stackentry
- if name[0].isalpha():
- # avoid to store things which are used just once
- self.picklenames[key] = name
+ try:
+ try:
+ meth = self.typecache[typ]
+ except KeyError:
+ obj_builtin_base = builtin_base(obj)
+ if (obj_builtin_base in (object,) + tuple(
+ self.simple_const_types.keys()) and
+ typ is not obj_builtin_base):
+ # assume it's a user defined thingy
+ meth = self.nameof_instance
+ else:
+ for cls in typ.__mro__:
+ meth = getattr(self, 'nameof_' + ''.join(
+ [ c for c in cls.__name__
+ if c.isalpha() or c == '_'] ), None)
+ if meth:
+ break
+ else:
+ raise Exception, "nameof(%r)" % (obj,)
+ self.typecache[typ] = meth
+ name = meth(obj)
+ except AlreadyCreated:
+ name = self.picklenames[id(obj)]
return name
def nameofargs(self, tup, plain_tuple = False):
""" a string with the nameofs, concatenated """
# see if we can build a compact representation
- for each in tup:
- if type(each) is tuple and len(each) > 2:
- break
- else:
- ret = ', '.join([self.nameof(arg) for arg in tup])
- if plain_tuple and len(tup) == 1:
- ret += ','
- if len(ret) <= 90:
- return ret
- ret = '\n ' + (',\n ').join(
+ ret = ', '.join([self.nameof(arg) for arg in tup])
+ if plain_tuple and len(tup) == 1:
+ ret += ','
+ if len(ret) <= 90:
+ return ret
+ ret = '\n ' + ',\n '.join(
[self.nameof(arg) for arg in tup]) + ',\n '
return ret
- def uniquename(self, basename):
- return self.namespace.uniquename(basename)
-
- def initcode_python(self, name, pyexpr):
- # generate init code that will evaluate the given Python expression
- #self.initcode.append("print 'setting up', %r" % name)
- self.initcode.append("%s = %s" % (name, pyexpr))
-
def nameof_object(self, value):
if type(value) is not object:
- raise Exception, "nameof(%r)" % (value,)
- name = self.uniquename('g_object')
- self.initcode_python(name, "object()")
+ raise Exception, "nameof(%r): type %s not object" % (
+ value, type(value).__name__)
+ name = self.memoize_unique(value, 'g_object')
+ self.produce('%s = object()' % name)
return name
def nameof_module(self, value):
# all allowed here, we reproduce ourselves
if self.is_app_domain(value.__name__):
- name = self.uniquename('gmod_%s' % value.__name__)
- self.initcode.append('%s = new.module(%r)\n'
- 'sys.modules[%r] = %s'% (
+ name = self.memoize_unique(value, 'gmod_%s' % value.__name__)
+ self.produce('%s = new.module(%r)\n'
+ 'sys.modules[%r] = %s'% (
name, value.__name__, value.__name__, name) )
def initmodule():
- for k, v in value.__dict__.items():
+ names = value.__dict__.keys()
+ names.sort()
+ for k in names:
try:
+ v = value.__dict__[k]
nv = self.nameof(v)
yield '%s.%s = %s' % (name, k, nv)
except PicklingError:
pass
- self.later(initmodule())
+ for line in initmodule():
+ self.produce(line)
else:
- name = self.uniquename(value.__name__)
- self.initcode_python(name, "__import__(%r)" % (value.__name__,))
+ name = self.memoize_unique(value, value.__name__)
+ self.produce('%s = __import__(%r)' % (name, value.__name__,))
return name
- def nameof_int(self, value):
- return repr(value)
-
- # we don't need to name the following const types.
- # the compiler folds the consts the same way as we do.
- # note that true pickling is more exact, here.
- nameof_long = nameof_float = nameof_bool = nameof_NoneType = nameof_int
- nameof_str = nameof_unicode = nameof_int
-
- def skipped_function(self, func):
+ def skipped_function(self, func, reason=None, _dummydict={}):
# Generates a placeholder for missing functions
# that raises an exception when called.
# The original code object is retained in an
# attribute '_skipped_code'
- name = self.uniquename('gskippedfunc_' + func.__name__)
- codename = self.nameof(func.func_code)
- self.initcode.append('def %s(*a,**k):\n'
- ' raise NotImplementedError' % name)
- self.initcode.append('%s._skipped_code = %s' % (name, codename) )
- return name
+ skipname = 'gskippedfunc_' + func.__name__
+ funcname = func.__name__
+ # need to handle this specially
+ if id(func) in self.picklenames:
+ raise AlreadyCreated
+ # generate code object before the skipped func (reads better)
+ func_code = getattr(func, 'func_code', None) # maybe builtin
+ self.nameof(func_code)
+ if reason:
+ text = 'skipped: %r, see _skipped_code attr: %s' % (
+ reason, funcname)
+ else:
+ text = 'skipped, see _skipped_code attr: %s' % funcname
+ def dummy(*args, **kwds):
+ raise NotImplementedError, text
+ skippedfunc = new.function(dummy.func_code, _dummydict, skipname, (),
+ dummy.func_closure)
+ skippedfunc._skipped_code = func_code
+ name = self.nameof(skippedfunc)
+ return self.memoize(func, name)
def nameof_staticmethod(self, sm):
# XXX XXX XXXX
func = sm.__get__(42.5)
- name = self.uniquename('gsm_' + func.__name__)
functionname = self.nameof(func)
- self.initcode_python(name, 'staticmethod(%s)' % functionname)
+ name = self.memoize_unique(sm, 'gsm_' + func.__name__)
+ self.produce('%s = staticmethod(%s)' % (name, functionname))
return name
def nameof_instancemethod(self, meth):
+ func = self.nameof(meth.im_func)
+ typ = self.nameof(meth.im_class)
if meth.im_self is None:
# no error checking here
- return self.nameof(meth.im_func)
+ name = self.memoize_unique(meth, 'gmeth_' + func)
+ self.produce('%s = %s.%s' % (name, typ, meth.__name__))
else:
ob = self.nameof(meth.im_self)
- func = self.nameof(meth.im_func)
- typ = self.nameof(meth.im_class)
- name = self.uniquename('gmeth_'+meth.im_func.__name__)
- self.initcode_python(name, 'new.instancemethod(%s, %s, %s)' % (
- func, ob, typ))
- return name
+ name = self.memoize_unique(meth, 'gumeth_'+ func)
+ self.produce('%s = new.instancemethod(%s, %s, %s)' % (
+ name, func, ob, typ))
+ return name
- # new version: save if we don't know
def should_translate_attr(self, pbc, attr):
ann = self.translator.annotator
if ann:
@@ -241,22 +301,31 @@
# builtin function
# where does it come from? Python2.2 doesn't have func.__module__
for modname, module in sys.modules.items():
- # here we don't ignore extension modules
+ # here we don't ignore extension modules, but it must be
+ # a builtin module
+ if not module: continue
+ if hasattr(module, '__file__'):
+ fname = module.__file__.lower()
+ pyendings = '.py', '.pyc', '.pyo'
+ if [fname.endswith(ending) for ending in pyendings]:
+ continue
if func is getattr(module, func.__name__, None):
break
else:
- raise Exception, '%r not found in any built-in module' % (func,)
- name = self.uniquename('gbltin_' + func.__name__)
+ #raise Exception, '%r not found in any built-in module' % (func,)
+ return self.skipped_function(
+ func, 'not found in any built-in module')
+ name = self.memoize_unique(func, 'gbltin_' + func.__name__)
if modname == '__builtin__':
- self.initcode_python(name, func.__name__)
+ self.produce('%s = %s' % (name, func.__name__))
else:
modname = self.nameof(module)
- self.initcode_python(name, '%s.%s' % (modname, func.__name__))
+ self.produce('%s = %s.%s' % (name, modname, func.__name__))
else:
# builtin (bound) method
- name = self.uniquename('gbltinmethod_' + func.__name__)
selfname = self.nameof(func.__self__)
- self.initcode_python(name, '%s.%s' % (selfname, func.__name__))
+ name = self.memoize_unique(func, 'gbltinmethod_' + func.__name__)
+ self.produce('%s = %s.%s' % (name, selfname, func.__name__))
return name
def nameof_classobj(self, cls):
@@ -265,27 +334,21 @@
try:
return self.save_global(cls)
- except PicklingError:
+ except PicklingError, e:
pass
metaclass = "type"
if issubclass(cls, Exception):
# if cls.__module__ == 'exceptions':
# don't rely on this, py.magic redefines AssertionError
- if getattr(__builtin__,cls.__name__,None) is cls:
- name = self.uniquename('gexc_' + cls.__name__)
- self.initcode_python(name, cls.__name__)
+ if getattr(__builtin__, cls.__name__, None) is cls:
+ name = self.memoize_unique(cls, 'gexc_' + cls.__name__)
+ self.produce('%s = %s' % (name, cls.__name__))
return name
- #else:
- # # exceptions must be old-style classes (grr!)
- # metaclass = "&PyClass_Type"
- # For the moment, use old-style classes exactly when the
- # pypy source uses old-style classes, to avoid strange problems.
if not isinstance(cls, type):
assert type(cls) is ClassType
metaclass = "types.ClassType"
- name = self.uniquename('gcls_' + cls.__name__)
basenames = [self.nameof(base) for base in cls.__bases__]
def initclassobj():
content = cls.__dict__.items()
@@ -302,7 +365,6 @@
if isapp:
if (isinstance(value, staticmethod) and value.__get__(1) not in
self.translator.flowgraphs and self.translator.frozen):
- print value
continue
if isinstance(value, classmethod):
doc = value.__get__(cls).__doc__
@@ -310,7 +372,6 @@
continue
if (isinstance(value, FunctionType) and value not in
self.translator.flowgraphs and self.translator.frozen):
- print value
continue
if key in ignore:
continue
@@ -322,15 +383,16 @@
baseargs = ", ".join(basenames)
if baseargs:
baseargs = '(%s)' % baseargs
+ name = self.memoize_unique(cls, 'gcls_' + cls.__name__)
ini = 'class %s%s:\n __metaclass__ = %s' % (name, baseargs, metaclass)
if '__slots__' in cls.__dict__:
ini += '\n __slots__ = %r' % cls.__slots__
- self.initcode.append(ini)
- self.initcode.append('%s.name = %r' % (name, cls.__name__))
- # squeeze it out, now# self.later(initclassobj())
- self.picklenames[Constant(cls)] = name
+ self.produce(ini)
+ self.produce('%s.__name__ = %r' % (name, cls.__name__))
+ self.produce('%s.__module__ = %r' % (name, cls.__module__))
+ # squeeze it out, early # self.later(initclassobj())
for line in initclassobj():
- self.initcode.append(line)
+ self.produce(line)
return name
nameof_class = nameof_classobj # for Python 2.2
@@ -357,6 +419,8 @@
type(type.__dict__['__basicsize__']): "type(type.__dict__['__basicsize__'])",
# type 'instancemethod':
type(Exception().__init__): 'type(Exception().__init__)',
+ # type 'listiterator':
+ type(iter([])): 'type(iter([]))',
}
descriptor_filter = {}
for _key in typename_mapping.keys():
@@ -367,46 +431,54 @@
def nameof_type(self, cls):
if cls.__module__ != '__builtin__':
return self.nameof_classobj(cls) # user-defined type
- name = self.uniquename('gtype_%s' % cls.__name__)
+ name = self.memoize_unique(cls, 'gtype_%s' % cls.__name__)
if getattr(__builtin__, cls.__name__, None) is cls:
expr = cls.__name__ # type available from __builtin__
elif cls in types.__dict__.values():
for key, value in types.__dict__.items():
if value is cls:
break
- self.initcode.append('from types import %s as %s' % (
+ self.produce('from types import %s as %s' % (
key, name))
return name
else:
expr = self.typename_mapping[cls]
- self.initcode_python(name, expr)
+ self.produce('%s = %s' % (name, expr))
return name
def nameof_tuple(self, tup):
chunk = 20
- name = self.uniquename('T%d' % len(tup))
+ # first create all arguments
+ for i in range(0, len(tup), chunk):
+ self.nameofargs(tup[i:i+chunk], True)
+ # see if someone else created us meanwhile
+ name = self.memoize_unique(tup, 'T%d' % len(tup))
argstr = self.nameofargs(tup[:chunk], True)
- self.initcode_python(name, '(%s)' % argstr)
+ self.produce('%s = (%s)' % (name, argstr))
for i in range(chunk, len(tup), chunk):
argstr = self.nameofargs(tup[i:i+chunk], True)
- self.initcode.append('%s += (%s)' % (name, argstr) )
+ self.produce('%s += (%s)' % (name, argstr) )
return name
def nameof_list(self, lis):
chunk = 20
- name = self.uniquename('L%d' % len(lis))
def initlist():
- chunk = 20
for i in range(0, len(lis), chunk):
argstr = self.nameofargs(lis[i:i+chunk])
yield '%s += [%s]' % (name, argstr)
- self.initcode_python(name, '[]')
- self.later(initlist())
+ name = self.memoize_unique(lis, 'L%d' % len(lis))
+ self.produce('%s = []' % name)
+ for line in initlist():
+ self.produce(line)
return name
def is_app_domain(self, modname):
for domain in self.domains:
- if modname.startswith(domain):
+ if domain.endswith('.') and modname.startswith(domain):
+ # handle subpaths
+ return True
+ if modname == domain:
+ # handle exact module names
return True
return False
@@ -414,6 +486,8 @@
if '__name__' in dic:
module = dic['__name__']
try:
+ if type(module) is str and self.is_app_domain(module):
+ raise ImportError
__import__(module)
mod = sys.modules[module]
except (ImportError, KeyError, TypeError):
@@ -421,51 +495,38 @@
else:
if dic is mod.__dict__ and not self.is_app_domain(module):
dictname = module.split('.')[-1] + '__dict__'
- dictname = self.uniquename(dictname)
- self.initcode.append('from %s import __dict__ as %s' % (
- module, dictname) )
- self.picklenames[Constant(dic)] = dictname
+ dictname = self.memoize_unique(dic, dictname)
+ self.produce('from %s import __dict__ as %s' % (
+ module, dictname) )
return dictname
- name = self.uniquename('D%d' % len(dic))
def initdict():
- for k in dic:
+ keys = dic.keys()
+ keys.sort()
+ for k in keys:
try:
- if type(k) is str:
- yield '%s[%r] = %s' % (name, k, self.nameof(dic[k]))
- else:
- yield '%s[%s] = %s' % (name, self.nameof(k),
- self.nameof(dic[k]))
+ nk, nv = self.nameof(k), self.nameof(dic[k])
+ yield '%s[%s] = %s' % (name, nk, nv)
except PicklingError:
pass
- self.initcode_python(name, '{}')
- self.later(initdict())
+ name = self.memoize_unique(dic, 'D%d' % len(dic))
+ self.produce('%s = {}' % name)
+ for line in initdict():
+ self.produce(line)
return name
# strange prebuilt instances below, don't look too closely
# XXX oh well.
def nameof_member_descriptor(self, md):
- name = self.uniquename('gdescriptor_%s_%s' % (
- md.__objclass__.__name__, md.__name__))
cls = self.nameof(md.__objclass__)
- self.initcode_python(name, '%s.__dict__[%r]' % (cls, md.__name__))
+ name = self.memoize_unique(md, 'gdescriptor_%s_%s' % (
+ md.__objclass__.__name__, md.__name__))
+ self.produce('%s = %s.__dict__[%r]' % (name, cls, md.__name__))
return name
nameof_getset_descriptor = nameof_member_descriptor
nameof_method_descriptor = nameof_member_descriptor
nameof_wrapper_descriptor = nameof_member_descriptor
def nameof_instance(self, instance):
- klass = instance.__class__
- if klass in self.shortnames:
- name = self.uniquename(self.shortnames[klass])
- else:
- name = self.uniquename('ginst_' + klass.__name__)
- cls = self.nameof(klass)
- if hasattr(klass, '__base__'):
- base_class = builtin_base(instance)
- base = self.nameof(base_class)
- else:
- base_class = None
- base = cls
def initinstance():
if hasattr(instance, '__setstate__'):
# the instance knows what to do
@@ -481,14 +542,33 @@
"%s has no dict and no __setstate__" % name)
content = restorestate.items()
content.sort()
+ attrs = []
for key, value in content:
if self.should_translate_attr(instance, key):
if hasattr(value, '__doc__'):
doc = value.__doc__
if type(doc) is str and doc.lstrip().startswith('NOT_RPYTHON'):
continue
- line = '%s.%s = %s' % (name, key, self.nameof(value))
- yield line
+ attrs.append( (key, self.nameof(value)) )
+ for k, v in attrs:
+ yield '%s.%s = %s' % (name, k, v)
+
+ klass = instance.__class__
+ cls = self.nameof(klass)
+ if hasattr(klass, '__base__'):
+ base_class = builtin_base(instance)
+ base = self.nameof(base_class)
+ else:
+ base_class = None
+ base = cls
+ if klass in self.inline_instances:
+ immediate = True
+ else:
+ if klass in self.shortnames:
+ name = self.memoize_unique(instance, self.shortnames[klass])
+ else:
+ name = self.memoize_unique(instance, 'ginst_' + klass.__name__)
+ immediate = False
if hasattr(instance, '__reduce_ex__'):
try:
reduced = instance.__reduce_ex__()
@@ -501,7 +581,7 @@
assert not hasattr(instance, '__dict__'), ('wrong assumptions'
' about __slots__ in %s instance without __setstate__,'
' please update %s' % (cls.__name__, __name__) )
- restorestate = slotted.__getstate__(instance)
+ restorestate = _get(instance)
restorer = _rec
restoreargs = klass,
else:
@@ -526,50 +606,50 @@
else:
restorestate = instance.__dict__
restoreargstr = self.nameofargs(restoreargs)
+ if immediate:
+ assert restorestate is None
+ return '%s(%s)' % (restorename, restoreargstr)
if isinstance(klass, type):
- self.initcode.append('%s = %s(%s)' % (name, restorename,
- restoreargstr))
+ self.produce('%s = %s(%s)' % (name, restorename, restoreargstr))
else:
- self.initcode.append('%s = new.instance(%s)' % (name, cls))
+ self.produce('%s = new.instance(%s)' % (name, cls))
if restorestate is not None:
- self.later(initinstance())
+ for line in initinstance():
+ self.produce(line)
return name
def save_global(self, obj):
# this is almost similar to pickle.py
name = obj.__name__
- key = Constant(obj)
- if key not in self.picklenames:
- module = getattr(obj, "__module__", None)
- if module is None:
- module = whichmodule(obj, name)
- if self.is_app_domain(module):
- # not allowed to import this
- raise PicklingError('%s belongs to the user program' %
- name)
- try:
- __import__(module)
- mod = sys.modules[module]
- klass = getattr(mod, name)
- except (ImportError, KeyError, AttributeError):
+ module = getattr(obj, "__module__", None)
+ if module is None:
+ module = whichmodule(obj, name)
+ if self.is_app_domain(module):
+ # not allowed to import this
+ raise PicklingError('%s belongs to the user program' %
+ name)
+ try:
+ __import__(module)
+ mod = sys.modules[module]
+ klass = getattr(mod, name)
+ except (ImportError, KeyError, AttributeError):
+ raise PicklingError(
+ "Can't pickle %r: it's not found as %s.%s" %
+ (obj, module, name))
+ else:
+ if klass is not obj:
raise PicklingError(
- "Can't pickle %r: it's not found as %s.%s" %
+ "Can't pickle %r: it's not the same object as %s.%s" %
(obj, module, name))
- else:
- if klass is not obj:
- raise PicklingError(
- "Can't pickle %r: it's not the same object as %s.%s" %
- (obj, module, name))
- # from here we do our own stuff
- restorename = self.uniquename(obj.__name__)
- if restorename != obj.__name__:
- self.initcode.append('from %s import %s as %s' % (
- module, obj.__name__, restorename) )
- else:
- self.initcode.append('from %s import %s' % (
- module, obj.__name__) )
- self.picklenames[key] = restorename
- return self.picklenames[key]
+ # from here we do our own stuff
+ restorename = self.memoize_unique(obj, obj.__name__)
+ if restorename != obj.__name__:
+ self.produce('from %s import %s as %s' % (
+ module, obj.__name__, restorename) )
+ else:
+ self.produce('from %s import %s' % (
+ module, obj.__name__) )
+ return restorename
def nameof_function(self, func):
# look for skipped functions
@@ -578,41 +658,50 @@
# see if this is in translator's domain
module = whichmodule(func, func.__name__)
if self.is_app_domain(module):
- return self.skipped_function(func)
+ # see if this buddy has been skipped in another save, before
+ if not hasattr(func, '_skipped_code'):
+ return self.skipped_function(func,
+ 'not found in translator\'s flowgraphs')
else:
if (func.func_doc and
func.func_doc.lstrip().startswith('NOT_RPYTHON')):
- return self.skipped_function(func)
+ return self.skipped_function(func, 'tagged as NOT_RPYTHON')
try:
return self.save_global(func)
except PicklingError:
pass
args = (func.func_code, func.func_globals, func.func_name,
func.func_defaults, func.func_closure)
- pyfuncobj = self.uniquename('gfunc_' + func.__name__)
- self.initcode.append('%s = new.function(%s)' % (pyfuncobj,
- self.nameofargs(args)) )
+ argstr = self.nameofargs(args)
+ if hasattr(func, '_skipped_code'):
+ name = self.memoize_unique(func, func.__name__)
+ else:
+ name = self.memoize_unique(func, 'gfunc_' + func.__name__)
+ self.produce('%s = new.function(%s)' % (name, argstr) )
if func.__dict__:
- for k, v in func.__dict__.items():
- try:
- self.initcode.append('%s.%s = %s' % (
- pyfuncobj, k, self.nameof(v)) )
- except PicklingError:
- pass
- return pyfuncobj
+ def initfunction():
+ items = func.__dict__.items()
+ items.sort()
+ for k, v in items:
+ try:
+ yield '%s.%s = %s' % (name, k, self.nameof(v))
+ except PicklingError:
+ pass
+ for line in initfunction():
+ self.produce(line)
+ return name
def nameof_cell(self, cel):
+ # no need to name cells. Their contents is what is shared.
obj = break_cell(cel)
- pycell = self.uniquename('gcell_' + self.nameof(obj))
- self.initcode.append('%s = %s(%s)' % (pycell, self.nameof(make_cell),
- self.nameof(obj)) )
- return pycell
+ return '%s(%s)' % (self.nameof(make_cell), self.nameof(obj))
def nameof_property(self, prop):
- pyprop = self.uniquename('gprop_')
- self.initcode.append('%s = property(%s)' % (pyprop, self.nameofargs(
- (prop.fget, prop.fset, prop.fdel, prop.__doc__))) )
- return pyprop
+ argstr = self.nameofargs((prop.fget, prop.fset, prop.fdel,
+ prop.__doc__))
+ name = self.memoize_unique(prop, 'gprop_')
+ self.produce('%s = property(%s)' % (name, argstr) )
+ return name
def nameof_code(self, code):
args = (code.co_argcount, code.co_nlocals, code.co_stacksize,
@@ -621,9 +710,9 @@
code.co_firstlineno, code.co_lnotab, code.co_freevars,
code.co_cellvars)
argstr = self.nameofargs(args)
- codeobj = self.uniquename('gcode_' + code.co_name)
- self.initcode.append('%s = new.code(%s)' % (codeobj, argstr))
- return codeobj
+ name = self.memoize_unique(code, 'gcode_' + code.co_name)
+ self.produce('%s = new.code(%s)' % (name, argstr))
+ return name
def nameof_file(self, fil):
if fil is sys.stdin: return "sys.stdin"
@@ -633,46 +722,10 @@
def nameof_methodwrapper(self, wp):
# this object should be enhanced in CPython!
- reprwp = repr(wp)
- name = wp.__name__
- def dummy_methodwrapper():
- return reprwp + (': method %s of unknown object '
- 'cannot be reconstructed, sorry!' % name )
- return self.nameof(dummy_methodwrapper)
-
- def later(self, gen):
- self.latercode.append((gen, self.debugstack))
+ msg = '%r: method %s of unknown object cannot be reconstructed' % (
+ wp, wp.__name__)
+ return self.skipped_function(wp, msg)
- def collect_initcode(self):
- writer = self.writer
- while self.latercode:
- gen, self.debugstack = self.latercode.pop()
- #self.initcode.extend(gen) -- eats TypeError! bad CPython!
- for line in gen:
- self.initcode.append(line)
- self.debugstack = ()
- if writer:
- for line in self.initcode:
- writer.write(line)
- del self.initcode[:]
- if writer:
- writer.close()
-
- def getfrozenbytecode(self):
- self.initcode.append('')
- source = '\n'.join(self.initcode)
- del self.initcode[:]
- co = compile(source, '<initcode>', 'exec')
- originalsource = source
- small = zlib.compress(marshal.dumps(co))
- source = """if 1:
- import zlib, marshal
- exec marshal.loads(zlib.decompress(%r))""" % small
- # Python 2.2 SyntaxError without newline: Bug #501622
- source += '\n'
- co = compile(source, '<initcode>', 'exec')
- del source
- return marshal.dumps(co), originalsource
def make_cell(obj):
def func():
@@ -693,5 +746,10 @@
def _rec(klass, base=object, state=None):
return _reconstructor(klass, base, state)
+def _get(obj):
+ return slotted.__getstate__(obj)
+
def _set(obj, *args):
slotted.__setstate__(obj, args)
+
+__all__ = ['GenPickle']
Modified: pypy/dist/pypy/translator/pickle/loader.py
==============================================================================
--- pypy/dist/pypy/translator/pickle/loader.py (original)
+++ pypy/dist/pypy/translator/pickle/loader.py Fri Jun 17 01:47:36 2005
@@ -2,6 +2,7 @@
class Loader:
def __init__(self, fname):
+ self.opened = False
self.f = self.open_file(fname)
def open_file(self, fname):
@@ -15,18 +16,22 @@
for blk in self.next_block():
exec blk in dic
try:
- return dic['ginst_Translator']
+ return dic['result']
finally:
self.close()
def close(self):
- self.f.close()
+ if self.opened:
+ self.f.close()
class TextLoader(Loader):
def open_file(self, fname):
- return file(fname)
+ if type(fname) is str:
+ self.opened = True
+ return file(fname)
+ return fname # should be a file-like object
def next_block(self):
data = self.f.read().split('## SECTION ##\n')
@@ -37,7 +42,10 @@
""" load compiled code from a ZIP file """
def open_file(self, fname):
- return zipfile.ZipFile(fname, "r")
+ if type(fname) is str:
+ self.opened = True
+ return zipfile.ZipFile(fname, "r")
+ return fname
def next_block(self):
root = self.f.read('root')
@@ -48,3 +56,5 @@
dump = self.f.read(name)
assert md5.new(dump).hexdigest() == name, "broken checksum"
yield marshal.loads(dump)
+
+__all__ = ['Loader', 'TextLoader', 'ZipLoader']
Modified: pypy/dist/pypy/translator/pickle/main.py
==============================================================================
--- pypy/dist/pypy/translator/pickle/main.py (original)
+++ pypy/dist/pypy/translator/pickle/main.py Fri Jun 17 01:47:36 2005
@@ -1,5 +1,37 @@
+import sys
+from pypy.translator.pickle.genpickle import GenPickle
+from pypy.translator.pickle.writer import Writer, TextWriter, ZipWriter
+from pypy.translator.pickle.loader import Loader, TextLoader, ZipLoader
+
def load(fname):
- pass
+ loader = _select(fname)[0]
+ assert loader, 'only .py and .zip files supported'
+ return loader(fname).load()
+
+def save(translator, fname, **objects):
+ writer = _select(fname)[1]
+ assert writer, 'only .py and .zip files supported'
+ assert objects, 'please provide objects to be saved as keywords'
+ pickler = GenPickle(translator, writer(fname))
+ hold = sys.getrecursionlimit()
+ if hold < 5000:
+ sys.setrecursionlimit(5000)
+ try:
+ pickler.pickle(**objects)
+ finally:
+ sys.setrecursionlimit(hold)
+ pickler.finish()
+ return pickler # for debugging purposes
+
+# and that's all, folks!
+# _________________________________________________________________
+
+def _select(fname):
+ name = fname.lower()
+ if name.endswith('.py'):
+ return TextLoader, TextWriter
+ elif name.endswith('.zip'):
+ return ZipLoader, ZipWriter
+ else:
+ return None, None
-def save(translator, fname):
- pass
Modified: pypy/dist/pypy/translator/pickle/writer.py
==============================================================================
--- pypy/dist/pypy/translator/pickle/writer.py (original)
+++ pypy/dist/pypy/translator/pickle/writer.py Fri Jun 17 01:47:36 2005
@@ -6,6 +6,7 @@
self.chunksize = 100000
self.count = 0
self.blocknum = 0
+ self.opened = False
self.f = self.open_file(fname)
def open_file(self, fname):
@@ -15,17 +16,18 @@
self.pieces.append(text)
self.count += len(text) + 1
if self.count >= self.chunksize:
- src = '\n'.join(self.pieces)
+ src = ''.join(self.pieces)
del self.pieces[:]
self.count -= self.chunksize
self.putblock(src)
self.blocknum += 1
def close(self):
- src = '\n'.join(self.pieces)
+ src = ''.join(self.pieces)
self.putblock(src)
self.finalize()
- self.f.close()
+ if self.opened:
+ self.f.close()
def finalize(self):
pass
@@ -34,10 +36,13 @@
class TextWriter(Writer):
def open_file(self, fname):
- return file(fname, 'w')
+ if type(fname) is str:
+ self.opened = True
+ return file(fname, 'w')
+ return fname # should be a file-like object
def putblock(self, src):
- print >> self.f, src
+ self.f.write(src)
print >> self.f, '## SECTION ##'
class ZipWriter(Writer):
@@ -48,7 +53,10 @@
self.blocknames = []
def open_file(self, fname):
- return zipfile.ZipFile(fname, "w", zipfile.ZIP_DEFLATED)
+ if type(fname) is str:
+ self.opened = True
+ return zipfile.ZipFile(fname, "w", zipfile.ZIP_DEFLATED)
+ return fname
def putblock(self, src):
cod = compile(src, 'block_%d' % self.blocknum, 'exec')
@@ -62,3 +70,5 @@
digest = md5.new(dump).hexdigest()
self.f.writestr(digest, dump)
self.f.writestr('root', digest)
+
+__all__ = ['Writer', 'TextWriter', 'ZipWriter']
More information about the Pypy-commit
mailing list