[py-svn] r37129 - in py/branch/config/py: . execnet execnet/testing test/rsession test/rsession/testing
fijal at codespeak.net
fijal at codespeak.net
Mon Jan 22 14:04:05 CET 2007
Author: fijal
Date: Mon Jan 22 14:04:00 2007
New Revision: 37129
Added:
py/branch/config/py/execnet/rsync.py
- copied unchanged from r37125, py/branch/config/py/test/rsession/rsync.py
py/branch/config/py/execnet/rsync_remote.py
- copied unchanged from r37125, py/branch/config/py/test/rsession/rsync_remote.py
py/branch/config/py/execnet/testing/test_rsync.py
- copied unchanged from r37125, py/branch/config/py/test/rsession/testing/test_rsync.py
Removed:
py/branch/config/py/test/rsession/rsync.py
py/branch/config/py/test/rsession/rsync_remote.py
py/branch/config/py/test/rsession/testing/test_rsync.py
Modified:
py/branch/config/py/__init__.py
py/branch/config/py/test/rsession/hostmanage.py
Log:
Move rsync to execnet and expose it as py.execnet.RSync
Modified: py/branch/config/py/__init__.py
==============================================================================
--- py/branch/config/py/__init__.py (original)
+++ py/branch/config/py/__init__.py Mon Jan 22 14:04:00 2007
@@ -91,6 +91,9 @@
'execnet.PopenGateway' : ('./execnet/register.py', 'PopenGateway'),
'execnet.SshGateway' : ('./execnet/register.py', 'SshGateway'),
+ # execnet scripts
+ 'execnet.RSync' : ('./execnet/rsync.py', 'RSync'),
+
# input-output helping
'io.dupfile' : ('./io/dupfile.py', 'dupfile'),
'io.FDCapture' : ('./io/capture.py', 'FDCapture'),
Modified: py/branch/config/py/test/rsession/hostmanage.py
==============================================================================
--- py/branch/config/py/test/rsession/hostmanage.py (original)
+++ py/branch/config/py/test/rsession/hostmanage.py Mon Jan 22 14:04:00 2007
@@ -5,7 +5,6 @@
from py.__.test.rsession.master import \
setup_slave, MasterNode
from py.__.test.rsession import report
-from py.__.test.rsession.rsync import RSync
class HostInfo(object):
""" Class trying to store all necessary attributes
@@ -39,11 +38,11 @@
def __ne__(self, other):
return not self == other
-class HostRSync(RSync):
+class HostRSync(py.execnet.RSync):
""" An rsync wrapper which filters out *~, .svn/ and *.pyc
"""
def __init__(self, rsync_roots):
- RSync.__init__(self, delete=True)
+ py.execnet.RSync.__init__(self, delete=True)
self.rsync_roots = rsync_roots
def filter(self, path):
Deleted: /py/branch/config/py/test/rsession/rsync.py
==============================================================================
--- /py/branch/config/py/test/rsession/rsync.py Mon Jan 22 14:04:00 2007
+++ (empty file)
@@ -1,137 +0,0 @@
-import py, os, stat, md5
-from Queue import Queue
-
-
-class RSync(object):
-
- def __init__(self, callback=None, **options):
- for name in options:
- assert name in ('delete')
- self.options = options
- self.callback = callback
- self.channels = {}
- self.receivequeue = Queue()
- self.links = []
-
- def filter(self, path):
- return True
-
- def add_target(self, gateway, destdir, finishedcallback=None):
- def itemcallback(req):
- self.receivequeue.put((channel, req))
- channel = gateway.remote_exec(REMOTE_SOURCE)
- channel.setcallback(itemcallback, endmarker = None)
- channel.send((str(destdir), self.options))
- self.channels[channel] = finishedcallback
-
- def send(self, sourcedir):
- self.sourcedir = str(sourcedir)
- # normalize a trailing '/' away
- self.sourcedir = os.path.dirname(os.path.join(self.sourcedir, 'x'))
- # send directory structure and file timestamps/sizes
- self._send_directory_structure(self.sourcedir)
-
- # paths and to_send are only used for doing
- # progress-related callbacks
- self.paths = {}
- self.to_send = {}
-
- # send modified file to clients
- while self.channels:
- channel, req = self.receivequeue.get()
- if req is None:
- # end-of-channel
- if channel in self.channels:
- # too early! we must have got an error
- channel.waitclose()
- # or else we raise one
- raise IOError('connection unexpectedly closed: %s ' % (
- channel.gateway,))
- else:
- command, data = req
- if command == "links":
- for link in self.links:
- channel.send(link)
- # completion marker, this host is done
- channel.send(42)
- elif command == "done":
- finishedcallback = self.channels.pop(channel)
- if finishedcallback:
- finishedcallback()
- elif command == "ack":
- if self.callback:
- self.callback("ack", self.paths[data], channel)
- elif command == "list_done":
- # sum up all to send
- if self.callback:
- s = sum([self.paths[i] for i in self.to_send[channel]])
- self.callback("list", s, channel)
- elif command == "send":
- modified_rel_path, checksum = data
- modifiedpath = os.path.join(self.sourcedir, *modified_rel_path)
- f = open(modifiedpath, 'rb')
- data = f.read()
-
- # provide info to progress callback function
- modified_rel_path = "/".join(modified_rel_path)
- self.paths[modified_rel_path] = len(data)
- if channel not in self.to_send:
- self.to_send[channel] = []
- self.to_send[channel].append(modified_rel_path)
-
- f.close()
- if checksum is not None and checksum == md5.md5(data).digest():
- data = None # not really modified
- else:
- # ! there is a reason for the interning:
- # sharing multiple copies of the file's data
- data = intern(data)
- print '%s <= %s' % (
- channel.gateway._getremoteaddress(),
- modified_rel_path)
- channel.send(data)
- del data
- else:
- assert "Unknown command %s" % command
-
- def _broadcast(self, msg):
- for channel in self.channels:
- channel.send(msg)
-
- def _send_link(self, basename, linkpoint):
- self.links.append(("link", basename, linkpoint))
-
- def _send_directory_structure(self, path):
- st = os.lstat(path)
- if stat.S_ISREG(st.st_mode):
- # regular file: send a timestamp/size pair
- self._broadcast((st.st_mtime, st.st_size))
- elif stat.S_ISDIR(st.st_mode):
- # dir: send a list of entries
- names = []
- subpaths = []
- for name in os.listdir(path):
- p = os.path.join(path, name)
- if self.filter(p):
- names.append(name)
- subpaths.append(p)
- self._broadcast(names)
- for p in subpaths:
- self._send_directory_structure(p)
- elif stat.S_ISLNK(st.st_mode):
- linkpoint = os.readlink(path)
- basename = path[len(self.sourcedir) + 1:]
- if not linkpoint.startswith(os.sep):
- # relative link, just send it
- self._send_link(basename, linkpoint)
- elif linkpoint.startswith(self.sourcedir):
- self._send_link(basename, linkpoint[len(self.sourcedir) + 1:])
- else:
- self._send_link(basename, linkpoint)
- self._broadcast(None)
- else:
- raise ValueError, "cannot sync %r" % (path,)
-
-REMOTE_SOURCE = py.path.local(__file__).dirpath().\
- join('rsync_remote.py').open().read() + "\nf()"
-
Deleted: /py/branch/config/py/test/rsession/rsync_remote.py
==============================================================================
--- /py/branch/config/py/test/rsession/rsync_remote.py Mon Jan 22 14:04:00 2007
+++ (empty file)
@@ -1,86 +0,0 @@
-
-def f():
- import os, stat, shutil, md5
- destdir, options = channel.receive()
- modifiedfiles = []
-
- def remove(path):
- assert path.startswith(destdir)
- try:
- os.unlink(path)
- except OSError:
- # assume it's a dir
- shutil.rmtree(path)
-
- def receive_directory_structure(path, relcomponents):
- try:
- st = os.lstat(path)
- except OSError:
- st = None
- msg = channel.receive()
- if isinstance(msg, list):
- if st and not stat.S_ISDIR(st.st_mode):
- os.unlink(path)
- st = None
- if not st:
- os.makedirs(path)
- entrynames = {}
- for entryname in msg:
- receive_directory_structure(os.path.join(path, entryname),
- relcomponents + [entryname])
- entrynames[entryname] = True
- if options.get('delete'):
- for othername in os.listdir(path):
- if othername not in entrynames:
- otherpath = os.path.join(path, othername)
- remove(otherpath)
- elif msg is not None:
- checksum = None
- if st:
- if stat.S_ISREG(st.st_mode):
- msg_mtime, msg_size = msg
- if msg_size != st.st_size:
- pass
- elif msg_mtime != st.st_mtime:
- f = open(path, 'rb')
- checksum = md5.md5(f.read()).digest()
- f.close()
- else:
- return # already fine
- else:
- remove(path)
- channel.send(("send", (relcomponents, checksum)))
- modifiedfiles.append((path, msg))
- receive_directory_structure(destdir, [])
-
- STRICT_CHECK = False # seems most useful this way for py.test
- channel.send(("list_done", None))
-
- for path, (time, size) in modifiedfiles:
- data = channel.receive()
- channel.send(("ack", path[len(destdir) + 1:]))
- if data is not None:
- if STRICT_CHECK and len(data) != size:
- raise IOError('file modified during rsync: %r' % (path,))
- f = open(path, 'wb')
- f.write(data)
- f.close()
- os.utime(path, (time, time))
- del data
- channel.send(("links", None))
-
- msg = channel.receive()
- while msg is not 42:
- # we get symlink
- _type, relpath, linkpoint = msg
- assert _type == "link"
- path = os.path.join(destdir, relpath)
- try:
- remove(path)
- except OSError:
- pass
-
- os.symlink(os.path.join(destdir, linkpoint), path)
- msg = channel.receive()
- channel.send(("done", None))
-
Deleted: /py/branch/config/py/test/rsession/testing/test_rsync.py
==============================================================================
--- /py/branch/config/py/test/rsession/testing/test_rsync.py Mon Jan 22 14:04:00 2007
+++ (empty file)
@@ -1,80 +0,0 @@
-import py
-from py.__.test.rsession.rsync import RSync
-
-
-def setup_module(mod):
- mod.gw = py.execnet.PopenGateway()
- mod.gw2 = py.execnet.PopenGateway()
-
-def teardown_module(mod):
- mod.gw.exit()
- mod.gw2.exit()
-
-
-def test_dirsync():
- base = py.test.ensuretemp('dirsync')
- dest = base.join('dest')
- dest2 = base.join('dest2')
- source = base.mkdir('source')
-
- for s in ('content1', 'content2-a-bit-longer'):
- source.ensure('subdir', 'file1').write(s)
- rsync = RSync()
- rsync.add_target(gw, dest)
- rsync.add_target(gw2, dest2)
- rsync.send(source)
- assert dest.join('subdir').check(dir=1)
- assert dest.join('subdir', 'file1').check(file=1)
- assert dest.join('subdir', 'file1').read() == s
- assert dest2.join('subdir').check(dir=1)
- assert dest2.join('subdir', 'file1').check(file=1)
- assert dest2.join('subdir', 'file1').read() == s
-
- source.join('subdir').remove('file1')
- rsync = RSync()
- rsync.add_target(gw2, dest2)
- rsync.add_target(gw, dest)
- rsync.send(source)
- assert dest.join('subdir', 'file1').check(file=1)
- assert dest2.join('subdir', 'file1').check(file=1)
- rsync = RSync(delete=True)
- rsync.add_target(gw2, dest2)
- rsync.add_target(gw, dest)
- rsync.send(source)
- assert not dest.join('subdir', 'file1').check()
- assert not dest2.join('subdir', 'file1').check()
-
-def test_symlink_rsync():
- if py.std.sys.platform == 'win32':
- py.test.skip("symlinks are unsupported on Windows.")
- base = py.test.ensuretemp('symlinkrsync')
- dest = base.join('dest')
- source = base.join('source')
- source.ensure("existant")
- source.join("rellink").mksymlinkto(source.join("existant"), absolute=0)
- source.join('abslink').mksymlinkto(source.join("existant"))
-
- rsync = RSync()
- rsync.add_target(gw, dest)
- rsync.send(source)
-
- assert dest.join('rellink').readlink() == dest.join("existant")
- assert dest.join('abslink').readlink() == dest.join("existant")
-
-def test_callback():
- base = py.test.ensuretemp('callback')
- dest = base.join("dest")
- source = base.join("source")
- source.ensure("existant").write("a" * 100)
- source.ensure("existant2").write("a" * 10)
- total = {}
- def callback(cmd, lgt, channel):
- total[(cmd, lgt)] = True
-
- rsync = RSync(callback=callback)
- #rsync = RSync()
- rsync.add_target(gw, dest)
- rsync.send(source)
-
- assert total == {("list", 110):True, ("ack", 100):True, ("ack", 10):True}
-
More information about the pytest-commit
mailing list