[Python-checkins] python/nondist/sandbox/mailbox mailbox.py, 1.8, 1.9 libmailbox.tex, 1.9, 1.10
gregorykjohnson@users.sourceforge.net
gregorykjohnson at users.sourceforge.net
Tue Aug 16 19:12:14 CEST 2005
Update of /cvsroot/python/python/nondist/sandbox/mailbox
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv5623
Modified Files:
mailbox.py libmailbox.tex
Log Message:
* Overhaul locking:
* Introduce lock() and unlock() methods:
* Locking granularity should be up to the user. Otherwise,
either consecutive operations that should be atomic are not or
locks grow stale (in as little as 30 seconds) and are stolen.
* Correct miscellaneous ill-conceived sections of locking code.
Index: mailbox.py
===================================================================
RCS file: /cvsroot/python/python/nondist/sandbox/mailbox/mailbox.py,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -d -r1.8 -r1.9
--- mailbox.py 13 Aug 2005 22:41:33 -0000 1.8
+++ mailbox.py 16 Aug 2005 17:12:03 -0000 1.9
@@ -169,6 +169,14 @@
"""Write any pending changes to the disk."""
raise NotImplementedError, "Method must be implemented by subclass"
+ def lock(self):
+ """Lock the mailbox."""
+ raise NotImplementedError, "Method must be implemented by subclass"
+
+ def unlock(self, f=None):
+ """Unlock the mailbox if it is locked."""
+ raise NotImplementedError, "Method must be implemented by subclass"
+
def close(self):
"""Flush and close the mailbox."""
raise NotImplementedError, "Method must be implemented by subclass"
@@ -315,6 +323,14 @@
"""Write any pending changes to disk."""
return # Maildir changes are always written immediately.
+ def lock(self):
+ """Lock the mailbox."""
+ return
+
+ def unlock(self, f=None):
+ """Unlock the mailbox if it is locked."""
+ return
+
def close(self):
"""Flush and close the mailbox."""
return
@@ -437,7 +453,7 @@
self._toc = None
self._next_key = 0
self._pending = False # No changes require rewriting the file.
- self._mtime = os.fstat(self._file.fileno()).st_mtime
+ self._locked = False
def add(self, message):
"""Add message and return assigned key."""
@@ -449,6 +465,7 @@
def remove(self, key):
"""Remove the keyed message; raise KeyError if it doesn't exist."""
+ self._lookup(key)
del self._toc[key]
self._pending = True
@@ -474,54 +491,59 @@
self._lookup()
return len(self._toc)
+ def lock(self):
+ """Lock the mailbox."""
+ if not self._locked:
+ _lock_file(self._file)
+ self._locked = True
+
+ def unlock(self, f=None):
+ """Unlock the mailbox if it is locked."""
+ if self._locked:
+ _unlock_file(self._file)
+ self._locked = False
+
def flush(self):
"""Write any pending changes to disk."""
if not self._pending:
return
self._lookup()
- _lock_file(self._file, self._path)
+ new_file = _create_temporary(self._path)
try:
- os.rename(self._path, self._path + '~')
- _lock_file(self._file, self._path + '~', system=False)
+ new_toc = {}
+ self._pre_mailbox_hook(new_file)
+ for key in sorted(self._toc.keys()):
+ start, stop = self._toc[key]
+ self._file.seek(start)
+ self._pre_message_hook(new_file)
+ new_start = new_file.tell()
+ while True:
+ buffer = self._file.read(min(4096,
+ stop - self._file.tell()))
+ if buffer == '':
+ break
+ new_file.write(buffer)
+ new_toc[key] = (new_start, new_file.tell())
+ self._post_message_hook(new_file)
except:
- _unlock_file(self._file, self._path)
+ new_file.close()
+ os.remove(new_file.name)
raise
+ new_file.close()
+ self._file.close()
try:
- self._assert_mtime()
- f = _create_carefully(self._path)
- try:
- _lock_file(f, self._path, dot=False)
- try:
- self._pre_mailbox_hook(f)
- new_toc = {}
- for key in sorted(self._toc.keys()):
- start, stop = self._toc[key][:2]
- self._file.seek(start)
- self._pre_message_hook(f)
- new_start = f.tell()
- while True:
- buffer = self._file.read(
- min(4096, stop - self._file.tell()))
- if buffer == '':
- break
- f.write(buffer)
- new_toc[key] = (new_start, f.tell()) + \
- self._toc[key][2:] # XXX: Wrong!
- self._post_message_hook(f)
- finally:
- _unlock_file(f, self._path)
- except:
- f.close()
+ os.rename(new_file.name, self._path)
+ except OSError, e:
+ if e.errno == errno.EEXIST:
+ os.remove(self._path)
+ os.rename(new_file.name, self._path)
+ else:
raise
- finally:
- _unlock_file(self._file, self._path + '~')
- self._file.close()
+ self._file = file(self._path, 'r+')
self._toc = new_toc
- self._file = f
- self._file.flush()
- self._set_mtime()
- os.remove(self._path + '~')
self._pending = False
+ if self._locked:
+ _lock_file(new_file, dotlock=False)
def _pre_mailbox_hook(self, f):
"""Called before writing the mailbox to file f."""
@@ -538,6 +560,8 @@
def close(self):
"""Flush and close the mailbox."""
self.flush()
+ if self._locked:
+ self.unlock()
self._file.close()
def _lookup(self, key=None):
@@ -550,39 +574,23 @@
except KeyError:
raise KeyError, "No message with key '%s'" % key
- def _assert_mtime(self):
- """Raise an exception if the file has been externally modified."""
- if self._mtime != os.fstat(self._file.fileno()).st_mtime:
- raise ExternalClashError, \
- 'External modifications detected: use refresh()'
-
- def _set_mtime(self):
- """Store the current mtime."""
- self._mtime = os.fstat(self._file.fileno()).st_mtime
-
def _append_message(self, message):
"""Append message to mailbox and return (start, stop, ...) offsets."""
- _lock_file(self._file, self._path)
- try:
- self._assert_mtime()
- self._file.seek(0, 2)
- self._pre_message_hook(self._file)
- offsets = self._install_message(message)
- self._post_message_hook(self._file)
- self._file.flush()
- self._set_mtime()
- finally:
- _unlock_file(self._file, self._path)
+ self._file.seek(0, 2)
+ self._pre_message_hook(self._file)
+ offsets = self._install_message(message)
+ self._post_message_hook(self._file)
+ self._file.flush()
return offsets
+
class _mboxMMDF(_singlefileMailbox):
"""An mbox or MMDF mailbox."""
def get_message(self, key):
"""Return a Message representation or raise a KeyError."""
start, stop = self._lookup(key)
- self._assert_mtime()
self._file.seek(start)
from_line = self._file.readline()
msg = self._message_factory(self._file.read(stop - self._file.tell()))
@@ -592,7 +600,6 @@
def get_string(self, key, from_=False):
"""Return a string representation or raise a KeyError."""
start, stop = self._lookup(key)
- self._assert_mtime()
self._file.seek(start)
if not from_:
self._file.readline()
@@ -601,7 +608,6 @@
def get_file(self, key, from_=False):
"""Return a file-like representation or raise a KeyError."""
start, stop = self._lookup(key)
- self._assert_mtime()
self._file.seek(start)
if not from_:
self._file.readline()
@@ -649,7 +655,6 @@
"""Generate key-to-(start, stop) table of contents."""
starts, stops = [], []
self._file.seek(0)
- self._assert_mtime()
while True:
line_pos = self._file.tell()
line = self._file.readline()
@@ -685,7 +690,6 @@
starts, stops = [], []
self._file.seek(0)
next_pos = 0
- self._assert_mtime()
while True:
line_pos = next_pos
line = self._file.readline()
@@ -718,9 +722,10 @@
if create:
os.mkdir(self._path, 0700)
os.close(os.open(os.path.join(self._path, '.mh_sequences'),
- os.O_CREAT | os.O_WRONLY, 0600))
+ os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0600))
else:
raise NoSuchMailboxError, self._path
+ self._locked = False
def add(self, message):
"""Add message and return assigned key."""
@@ -732,62 +737,85 @@
new_path = os.path.join(self._path, str(new_key))
f = _create_carefully(new_path)
try:
- _lock_file(f, f.name)
+ if self._locked:
+ _lock_file(f)
try:
self._dump_message(message, f)
if isinstance(message, MHMessage):
self._dump_sequences(message, new_key)
finally:
- _unlock_file(f, f.name)
+ if self._locked:
+ _unlock_file(f)
finally:
f.close()
return new_key
def remove(self, key):
"""Remove the keyed message; raise KeyError if it doesn't exist."""
+ path = os.path.join(self._path, str(key))
try:
- os.remove(os.path.join(self._path, str(key)))
- except OSError, e:
+ f = file(path, 'r+')
+ except IOError, e:
if e.errno == errno.ENOENT:
raise KeyError, "No message with key '%s'" % key
else:
raise
+ try:
+ if self._locked:
+ _lock_file(f)
+ try:
+ f.close()
+ os.remove(os.path.join(self._path, str(key)))
+ finally:
+ if self._locked:
+ _unlock_file(f)
+ finally:
+ f.close()
def __setitem__(self, key, message):
"""Replace the keyed message; raise KeyError if it doesn't exist."""
+ path = os.path.join(self._path, str(key))
try:
- f_r = file(os.path.join(self._path, str(key)), 'r+')
+ f = file(path, 'r+')
except IOError, e:
if e.errno == errno.ENOENT:
raise KeyError, "No message with key '%s'" % key
else:
raise
try:
- _lock_file(f_r, f_r.name)
+ if self._locked:
+ _lock_file(f)
try:
- f_w = file(os.path.join(self._path, str(key)), 'w')
- try:
- self._dump_message(message, f_w)
- if isinstance(message, MHMessage):
- self._dump_sequences(message, key)
- finally:
- f_w.close()
+ os.close(os.open(path, os.O_WRONLY | os.O_TRUNC))
+ self._dump_message(message, f)
+ if isinstance(message, MHMessage):
+ self._dump_sequences(message, key)
finally:
- _unlock_file(f_r, f_r.name)
+ if self._locked:
+ _unlock_file(f)
finally:
- f_r.close()
+ f.close()
def get_message(self, key):
"""Return a Message representation or raise a KeyError."""
try:
- f = file(os.path.join(self._path, str(key)), 'r')
+ if self._locked:
+ f = file(os.path.join(self._path, str(key)), 'r+')
+ else:
+ f = file(os.path.join(self._path, str(key)), 'r')
except IOError, e:
if e.errno == errno.ENOENT:
raise KeyError, "No message with key '%s'" % key
else:
raise
try:
- msg = MHMessage(f)
+ if self._locked:
+ _lock_file(f)
+ try:
+ msg = MHMessage(f)
+ finally:
+ if self._locked:
+ _unlock_file(f)
finally:
f.close()
for name, key_list in self.get_sequences():
@@ -798,14 +826,23 @@
def get_string(self, key):
"""Return a string representation or raise a KeyError."""
try:
- f = file(os.path.join(self._path, str(key)), 'r')
+ if self._locked:
+ f = file(os.path.join(self._path, str(key)), 'r+')
+ else:
+ f = file(os.path.join(self._path, str(key)), 'r')
except IOError, e:
if e.errno == errno.ENOENT:
raise KeyError, "No message with key '%s'" % key
else:
raise
try:
- return f.read()
+ if self._locked:
+ _lock_file(f)
+ try:
+ return f.read()
+ finally:
+ if self._locked:
+ _unlock_file()
finally:
f.close()
@@ -833,13 +870,29 @@
"""Return a count of messages in the mailbox."""
return len(list(self.iterkeys()))
+ def lock(self):
+ """Lock the mailbox."""
+ if not self._locked:
+ self._file = file(os.path.join(self._path, '.mh_sequences'), 'r+')
+ _lock_file(self._file)
+ self._locked = True
+
+ def unlock(self, f=None):
+ """Unlock the mailbox if it is locked."""
+ if self._locked:
+ _unlock_file(self._file)
+ self._file.close()
+ del self._file
+ self._locked = False
+
def flush(self):
"""Write any pending changes to the disk."""
return
def close(self):
"""Flush and close the mailbox."""
- return
+ if self._locked:
+ self.unlock()
def list_folders(self):
"""Return a list of folder names."""
@@ -855,7 +908,7 @@
def add_folder(self, folder):
"""Create a folder and return an MH instance representing it."""
- return Maildir(os.path.join(self._path, '.' + folder))
+ return MH(os.path.join(self._path, '.' + folder))
def remove_folder(self, folder):
"""Delete the named folder, which must be empty."""
@@ -899,36 +952,28 @@
"""Set sequences using the given name-to-key-list dictionary."""
f = file(os.path.join(self._path, '.mh_sequences'), 'r+')
try:
- _lock_file(f, f.name)
- try:
- # .truncate() is not portable, so re-open to truncate.
- f_w = file(os.path.join(self._path, '.mh_sequences'), 'w')
- try:
- for name, keys in sequences.iteritems():
- if len(keys) == 0:
- continue
- f_w.write('%s:' % name)
- prev = None
+ os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC))
+ for name, keys in sequences.iteritems():
+ if len(keys) == 0:
+ continue
+ f.write('%s:' % name)
+ prev = None
+ completing = False
+ for key in sorted(set(keys)):
+ if key - 1 == prev:
+ if not completing:
+ completing = True
+ fw.write('-')
+ elif completing:
completing = False
- for key in sorted(set(keys)):
- if key - 1 == prev:
- if not completing:
- completing = True
- f_w.write('-')
- elif completing:
- completing = False
- f_w.write('%s %s' % (prev, key))
- else:
- f_w.write(' %s' % key)
- prev = key
- if completing:
- f_w.write('%s%s' % (prev, os.linesep))
- else:
- f_w.write(os.linesep)
- finally:
- f_w.close()
- finally:
- _unlock_file(f, f.name)
+ f.write('%s %s' % (prev, key))
+ else:
+ f.write(' %s' % key)
+ prev = key
+ if completing:
+ f.write('%s%s' % (prev, os.linesep))
+ else:
+ f.write(os.linesep)
finally:
f.close()
@@ -940,13 +985,24 @@
for key in self.iterkeys():
if key - 1 != prev:
changes.append((key, prev + 1))
- if hasattr(os, 'link'):
- os.link(os.path.join(self._path, str(key)),
- os.path.join(self._path, str(prev + 1)))
- os.unlink(os.path.join(self._path, str(key)))
- else:
- os.rename(os.path.join(self._path, str(key)),
- os.path.join(self._path, str(prev + 1)))
+ f = file(os.path.join(self._path, str(key)), 'r+')
+ try:
+ if self._locked:
+ _lock_file(f)
+ try:
+ if hasattr(os, 'link'):
+ os.link(os.path.join(self._path, str(key)),
+ os.path.join(self._path, str(prev + 1)))
+ os.unlink(os.path.join(self._path, str(key)))
+ else:
+ f.close()
+ os.rename(os.path.join(self._path, str(key)),
+ os.path.join(self._path, str(prev + 1)))
+ finally:
+ if self._locked:
+ _unlock_file(f)
+ finally:
+ f.close()
prev += 1
self._next_key = prev + 1
if len(changes) == 0:
@@ -979,7 +1035,6 @@
def get_message(self, key):
"""Return a Message representation or raise a KeyError."""
start, stop = self._lookup(key)
- self._assert_mtime()
self._file.seek(start)
self._file.readline() # XXX: parse this '1,' line for labels
original_headers = StringIO.StringIO()
@@ -1002,7 +1057,6 @@
def get_string(self, key):
"""Return a string representation or raise a KeyError."""
start, stop = self._lookup(key)
- self._assert_mtime()
self._file.seek(start)
self._file.readline() # Skip '1,' line.
original_headers = StringIO.StringIO()
@@ -1031,7 +1085,6 @@
starts, stops = [], []
self._file.seek(0)
next_pos = 0
- self._assert_mtime()
while True:
line_pos = next_pos
line = self._file.readline()
@@ -1614,35 +1667,30 @@
return _ProxyFile._read(self, size, read_method)
-def _lock_file(f, path, system=True, dot=True):
+def _lock_file(f, dotlock=True):
"""Lock file f using lockf, flock, and dot locking."""
- lockf_done = flock_done = dotlock_done = False
try:
- if system and fcntl:
+ if fcntl:
try:
fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
- lockf_done = True
except IOError, e:
if e.errno == errno.EAGAIN:
raise ExternalClashError, \
- "lockf: lock unavailable: %s" % path
+ "lockf: lock unavailable: %s" % f.name
else:
raise
try:
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
- flock_done = True
except IOError, e:
if e.errno == errno.EWOULDBLOCK:
raise ExternalClashError, \
- "flock: lock unavailable: %s" % path
+ "flock: lock unavailable: %s" % f.name
else:
raise
- if dot:
- tmp = '%s.lock.%s.%s.%s' % (path, int(time.time()),
- socket.gethostname(), os.getpid())
+ if dotlock:
try:
- os.close(os.open(tmp, os.O_WRONLY | os.O_EXCL | os.O_CREAT,
- 0600))
+ pre_lock = _create_temporary(f.name + '.lock')
+ pre_lock.close()
except IOError, e:
if e.errno == errno.EACCESS:
return # Without write access, just skip dotlocking.
@@ -1650,40 +1698,48 @@
raise
try:
if hasattr(os, 'link'):
- os.link(tmp, path + '.lock')
- os.unlink(tmp)
+ os.link(pre_lock.name, f.name + '.lock')
+ dotlock_done = True
+ os.unlink(pre_lock)
else:
- os.rename(tmp, path + '.lock')
- dotlock_done = True
+ os.rename(pre_lock, f.name + '.lock')
+ dotlock_done = True
except OSError, e:
if e.errno == errno.EEXIST:
- try:
- os.remove(tmp)
- except:
- pass
- raise ExternalClashError, 'dot lock unavailable: %s' % path
+ os.remove(pre_lock)
+ raise ExternalClashError, 'dot lock unavailable: %s' % \
+ f.name
else:
raise
except:
- if lockf_done:
+ if fcntl:
fcntl.lockf(f, fcntl.LOCK_UN)
- if flock_done:
fcntl.flock(f, fcntl.LOCK_UN)
if dotlock_done:
- os.remove(path + '.lock')
+ os.remove(f.name + '.lock')
raise
-def _unlock_file(f, path, system=True, dot=True):
+def _unlock_file(f):
"""Unlock file f using lockf, flock, and dot locking."""
- if system and fcntl:
+ if fcntl:
fcntl.lockf(f, fcntl.LOCK_UN)
fcntl.flock(f, fcntl.LOCK_UN)
- if dot and os.path.exists(path + '.lock'):
- os.remove(path + '.lock')
+ if os.path.exists(path + '.lock'):
+ os.remove(f.name + '.lock')
def _create_carefully(path):
"""Create a file if it doesn't exist and open for reading and writing."""
- return os.fdopen(os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR), 'r+')
+ fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR)
+ try:
+ return file(path, 'r+')
+ finally:
+ os.close(fd)
+
+def _create_temporary(path):
+ """Create a temp file based on path and open for reading and writing."""
+ return _create_carefully('%s.%s.%s.%s' % (path, int(time.time()),
+ socket.gethostname(),
+ os.getpid()))
class Error(Exception):
"""Raised for module-specific errors."""
Index: libmailbox.tex
===================================================================
RCS file: /cvsroot/python/python/nondist/sandbox/mailbox/libmailbox.tex,v
retrieving revision 1.9
retrieving revision 1.10
diff -u -d -r1.9 -r1.10
--- libmailbox.tex 13 Aug 2005 22:41:33 -0000 1.9
+++ libmailbox.tex 16 Aug 2005 17:12:03 -0000 1.10
@@ -221,9 +221,20 @@
subclasses, changes are written immediately and this method does nothing.
\end{methoddesc}
+\begin{methoddesc}{lock}{}
+Acquire an exclusive advisory lock on the mailbox so that other processes know
+not to modify it. An \exception{ExternalClashError} is raised if the lock is
+not available. The particular locking mechanisms used depend upon the mailbox
+format.
+\end{methoddesc}
+
+\begin{methoddesc}{unlock}{}
+Release the advisory lock on the mailbox, if any.
+\end{methoddesc}
+
\begin{methoddesc}{close}{}
-Flush the mailbox and close any open files. For some \class{Mailbox}
-subclasses, this method does nothing.
+Flush the mailbox, unlock it if necessary, and close any open files. For some
+\class{Mailbox} subclasses, this method does nothing.
\end{methoddesc}
@@ -305,9 +316,14 @@
nothing.
\end{methoddesc}
+\begin{methoddesc}{lock}{}
+\methodline{unlock}{}
+Maildir mailboxes do not support (or require) locking, so these methods do
+nothing. \end{methoddesc}
+
\begin{methoddesc}{close}{}
-\class{Maildir} instances do not keep any open files, so this method does
-nothing.
+\class{Maildir} instances do not keep any open files and the underlying
+mailboxes do not support locking, so this method does nothing.
\end{methoddesc}
\begin{methoddesc}{get_file}{key}
@@ -356,6 +372,12 @@
XXX
\end{methoddesc}
+\begin{methoddesc}{lock}{}
+\methodline{unlock}{}
+Three locking mechanisms are used---dot locking and, if available, the
+\cfunction{flock()} and \cfunction{lockf()} system calls.
+\end{methoddesc}
+
\begin{seealso}
\seelink{http://www.qmail.org/man/man5/mbox.html}{mbox man page from
qmail}{A specification of the format and its variations.}
@@ -450,6 +472,15 @@
marking a message for deletion by prepending a comma to its name is not used.
\end{methoddesc}
+\begin{methoddesc}{lock}{}
+\methodline{unlock}{}
+Three locking mechanisms are used---dot locking and, if available, the
+\cfunction{flock()} and \cfunction{lockf()} system calls. For MH mailboxes,
+locking the mailbox means locking the \file{.mh_sequences} file and, only for
+the duration of any operations that affect them, locking individual message
+files.
+\end{methoddesc}
+
\begin{methoddesc}{get_file}{key}
XXX
\end{methoddesc}
@@ -460,7 +491,8 @@
\end{methoddesc}
\begin{methoddesc}{close}{}
-\class{MH} instances do not keep any open files, so this method does nothing.
+\class{MH} instances do not keep any open files, so this method is equivelant
+to \method{unlock()}.
\end{methoddesc}
\class{MH} instances have all of the methods of \class{Mailbox} in addition to
@@ -521,6 +553,12 @@
module), which may be used like a file.
\end{methoddesc}
+\begin{methoddesc}{lock}{}
+\methodline{unlock}{}
+Three locking mechanisms are used---dot locking and, if available, the
+\cfunction{flock()} and \cfunction{lockf()} system calls.
+\end{methoddesc}
+
\begin{seealso}
\seelink{http://quimby.gnus.org/notes/BABYL}{Format of Version 5 Babyl Files}{A
specification of the Babyl format.}
@@ -554,6 +592,12 @@
XXX
\end{methoddesc}
+\begin{methoddesc}{lock}{}
+\methodline{unlock}{}
+Three locking mechanisms are used---dot locking and, if available, the
+\cfunction{flock()} and \cfunction{lockf()} system calls.
+\end{methoddesc}
+
\begin{seealso}
\seelink{http://www.tin.org/bin/man.cgi?section=5\&topic=mmdf}{mmdf man page
from tin}{A specification of MMDF format from the documentation of tin, a
More information about the Python-checkins
mailing list