[pypy-svn] r34600 - in pypy/dist/pypy/lib: . test2
fijal at codespeak.net
fijal at codespeak.net
Tue Nov 14 16:32:25 CET 2006
Author: fijal
Date: Tue Nov 14 16:32:22 2006
New Revision: 34600
Added:
pypy/dist/pypy/lib/distributed.py (contents, props changed)
pypy/dist/pypy/lib/test2/test_distributed.py (contents, props changed)
Log:
First concept how I see distributed access (works locally anyway)
Added: pypy/dist/pypy/lib/distributed.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/lib/distributed.py Tue Nov 14 16:32:22 2006
@@ -0,0 +1,129 @@
+
+""" Distributed controller(s) for use with transparent proxy objects
+
+First idea:
+
+1. We use py.execnet to create a connection to wherever
+2. We run some code there (RSync in advance makes some sense)
+3. We access remote objects like normal ones, with a special protocol
+
+Local side:
+ - Request an object from remote side from global namespace as simple
+ --- request(name) --->
+ - Receive an object which is in protocol described below which is
+ constructed as shallow copy of the remote type.
+
+ Shallow copy is defined as follows:
+
+ - for interp-level object that we know we can provide transparent proxy
+ we just do that
+
+ - for others we fake or fail depending on object
+
+ - for user objects, we create a class which fakes all attributes of
+ a class as transparent proxies of remote objects, we create an instance
+ of that class and populate __dict__
+
+ - for immutable types, we just copy that
+
+Remote side:
+ - we run code, whatever we like
+ - additionally, we've got thread exporting stuff (or just exporting
+ globals, whatever)
+ - for every object, we just send an object, or provide a protocol for
+ sending it in a different way.
+
+"""
+
+try:
+ from pypymagic import transparent_proxy as proxy
+except ImportError:
+ raise ImportError("Cannot work without transparent proxy functional")
+
+# XXX We do not make any garbage collection. We'll need it at some point
+
+import types
+
+class AbstractProtocol(object):
+ letter_types = {
+ 'l' : list,
+ 'd' : dict,
+ 't' : tuple,
+ 'i' : int,
+ 'f' : float,
+ 'u' : unicode,
+ 'l' : long,
+ 's' : str,
+ 'lst' : list,
+ 'fun' : types.FunctionType
+ }
+ type_letters = dict([(value, key) for key, value in letter_types.items()])
+ assert len(type_letters) == len(letter_types)
+
+ def __init__(self):
+ self.objs = [] # we just store everything, maybe later
+ # we'll need some kind of garbage collection
+
+ def register_obj(self, obj):
+ self.objs.append(obj)
+ return len(self.objs) - 1
+
+ def wrap(self, obj):
+ """ Wrap an object as sth prepared for sending
+ """
+ tp = type(obj)
+ if tp in (str, int, float, long, unicode):
+ # simple, immutable object, just copy
+ return (self.type_letters[tp], obj)
+ elif tp is tuple:
+ # we just pack all of the items
+ return ('t', tuple([self.wrap(elem) for elem in obj]))
+ elif tp in (list, dict, types.FunctionType):
+ id = self.register_obj(obj)
+ return (self.type_letters[tp], id)
+ else:
+ raise NotImplementedError("Cannot wrap %s: unsupported type %s" %
+ (obj, tp))
+
+ def unwrap(self, data):
+ """ Unwrap an object
+ """
+ tp_letter, obj_data = data
+ tp = self.letter_types[tp_letter]
+ if tp in (str, int, float, long, unicode):
+ return obj_data # this is the object
+ elif tp is tuple:
+ return tuple([self.unwrap(i) for i in obj_data])
+ elif tp in (list, dict, types.FunctionType):
+ return proxy(tp, RemoteObject(self, obj_data).perform)
+ else:
+ raise NotImplementedError("Cannot unwrap %s" % (data,))
+
+ def perform(self, *args, **kwargs):
+ raise NotImplementedError("Abstract only protocol")
+
+class LocalProtocol(AbstractProtocol):
+ """ This is stupid protocol for testing purposes only
+ """
+ def perform(self, id, name, *args, **kwargs):
+ obj = self.objs[id]
+ return getattr(obj, name)(*args, **kwargs)
+
+class RemoteObject(object):
+ def __init__(self, protocol, id):
+ self.id = id
+ self.protocol = protocol
+
+ def perform(self, name, *args, **kwargs):
+ return self.protocol.perform(self.id, name, *args, **kwargs)
+
+##class RemoteFunction(object):
+## def __init__(self, channel, name):
+## channel.send(protocol.get(name))
+## self.channel = channel
+## self.id = protocol.id(channel.receive())
+## self.fun = proxy(types.FunctionType, self.perform)
+##
+## def perform(self, name, *args, **kwargs):
+## self.channel.send(protocol.pack_call(self.id, name, args, kwargs))
+## return protocol.unpack(self.channel.receive())
Added: pypy/dist/pypy/lib/test2/test_distributed.py
==============================================================================
--- (empty file)
+++ pypy/dist/pypy/lib/test2/test_distributed.py Tue Nov 14 16:32:22 2006
@@ -0,0 +1,43 @@
+
+""" Controllers tests
+"""
+
+from pypy.conftest import gettestobjspace
+
+class AppTestNoProxy(object):
+ disabled = True
+ def test_init(self):
+ raises(ImportError, "import distributed")
+
+class AppTestDistributed(object):
+ def setup_class(cls):
+ cls.space = gettestobjspace(**{"objspace.std.withtproxy": True})
+
+ def test_init(self):
+ import distributed
+ distributed.proxy
+
+ def test_protocol(self):
+ from distributed import AbstractProtocol
+ protocol = AbstractProtocol()
+ for item in ("aaa", 3, u"aa", 344444444444444444L, 1.2, (1, "aa")):
+ assert protocol.unwrap(protocol.wrap(item)) == item
+ assert type(protocol.unwrap(protocol.wrap([1,2,3]))) is list
+ assert type(protocol.unwrap(protocol.wrap({"a":3}))) is dict
+
+ def f():
+ pass
+
+ assert type(protocol.unwrap(protocol.wrap(f))) is type(f)
+
+ def test_protocol_run(self):
+ l = [1,2,3]
+ from distributed import LocalProtocol
+ protocol = LocalProtocol()
+ wrap = protocol.wrap
+ unwrap = protocol.unwrap
+ item = unwrap(wrap(l))
+ assert len(item) == 3
+ assert item[2] == 3
+ item += [1,1,1]
+ assert len(item) == 6
More information about the Pypy-commit
mailing list