[pypy-commit] pypy win32-cleanup2: merge win32-stdlib branch into branch
mattip
noreply at buildbot.pypy.org
Sun Apr 22 22:32:49 CEST 2012
Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54628:0550049021e6
Date: 2012-04-22 23:32 +0300
http://bitbucket.org/pypy/pypy/changeset/0550049021e6/
Log: merge win32-stdlib branch into branch
diff --git a/lib-python/2.7/mailbox.py b/lib-python/2.7/mailbox.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/mailbox.py
@@ -0,0 +1,2171 @@
+#! /usr/bin/env python
+
+"""Read/write support for Maildir, mbox, MH, Babyl, and MMDF mailboxes."""
+
+# Notes for authors of new mailbox subclasses:
+#
+# Remember to fsync() changes to disk before closing a modified file
+# or returning from a flush() method. See functions _sync_flush() and
+# _sync_close().
+
+import sys
+import os
+import time
+import calendar
+import socket
+import errno
+import copy
+import email
+import email.message
+import email.generator
+import StringIO
+try:
+ if sys.platform == 'os2emx':
+ # OS/2 EMX fcntl() not adequate
+ raise ImportError
+ import fcntl
+except ImportError:
+ fcntl = None
+
+import warnings
+with warnings.catch_warnings():
+ if sys.py3kwarning:
+ warnings.filterwarnings("ignore", ".*rfc822 has been removed",
+ DeprecationWarning)
+ import rfc822
+
+__all__ = [ 'Mailbox', 'Maildir', 'mbox', 'MH', 'Babyl', 'MMDF',
+ 'Message', 'MaildirMessage', 'mboxMessage', 'MHMessage',
+ 'BabylMessage', 'MMDFMessage', 'UnixMailbox',
+ 'PortableUnixMailbox', 'MmdfMailbox', 'MHMailbox', 'BabylMailbox' ]
+
+class Mailbox:
+ """A group of messages in a particular place."""
+
+ def __init__(self, path, factory=None, create=True):
+ """Initialize a Mailbox instance."""
+ self._path = os.path.abspath(os.path.expanduser(path))
+ self._factory = factory
+
+ def add(self, message):
+ """Add message and return assigned key."""
+ raise NotImplementedError('Method must be implemented by subclass')
+
+ def remove(self, key):
+ """Remove the keyed message; raise KeyError if it doesn't exist."""
+ raise NotImplementedError('Method must be implemented by subclass')
+
+ def __delitem__(self, key):
+ self.remove(key)
+
+ def discard(self, key):
+ """If the keyed message exists, remove it."""
+ try:
+ self.remove(key)
+ except KeyError:
+ pass
+
+ def __setitem__(self, key, message):
+ """Replace the keyed message; raise KeyError if it doesn't exist."""
+ raise NotImplementedError('Method must be implemented by subclass')
+
+ def get(self, key, default=None):
+ """Return the keyed message, or default if it doesn't exist."""
+ try:
+ return self.__getitem__(key)
+ except KeyError:
+ return default
+
+ def __getitem__(self, key):
+ """Return the keyed message; raise KeyError if it doesn't exist."""
+ if not self._factory:
+ return self.get_message(key)
+ else:
+ return self._factory(self.get_file(key))
+
+ def get_message(self, key):
+ """Return a Message representation or raise a KeyError."""
+ raise NotImplementedError('Method must be implemented by subclass')
+
+ def get_string(self, key):
+ """Return a string representation or raise a KeyError."""
+ raise NotImplementedError('Method must be implemented by subclass')
+
+ def get_file(self, key):
+ """Return a file-like representation or raise a KeyError."""
+ raise NotImplementedError('Method must be implemented by subclass')
+
+ def iterkeys(self):
+ """Return an iterator over keys."""
+ raise NotImplementedError('Method must be implemented by subclass')
+
+ def keys(self):
+ """Return a list of keys."""
+ return list(self.iterkeys())
+
+ def itervalues(self):
+ """Return an iterator over all messages."""
+ for key in self.iterkeys():
+ try:
+ value = self[key]
+ except KeyError:
+ continue
+ yield value
+
+ def __iter__(self):
+ return self.itervalues()
+
+ def values(self):
+ """Return a list of messages. Memory intensive."""
+ return list(self.itervalues())
+
+ def iteritems(self):
+ """Return an iterator over (key, message) tuples."""
+ for key in self.iterkeys():
+ try:
+ value = self[key]
+ except KeyError:
+ continue
+ yield (key, value)
+
+ def items(self):
+ """Return a list of (key, message) tuples. Memory intensive."""
+ return list(self.iteritems())
+
+ def has_key(self, key):
+ """Return True if the keyed message exists, False otherwise."""
+ raise NotImplementedError('Method must be implemented by subclass')
+
+ def __contains__(self, key):
+ return self.has_key(key)
+
+ def __len__(self):
+ """Return a count of messages in the mailbox."""
+ raise NotImplementedError('Method must be implemented by subclass')
+
+ def clear(self):
+ """Delete all messages."""
+ for key in self.iterkeys():
+ self.discard(key)
+
+ def pop(self, key, default=None):
+ """Delete the keyed message and return it, or default."""
+ try:
+ result = self[key]
+ except KeyError:
+ return default
+ self.discard(key)
+ return result
+
+ def popitem(self):
+ """Delete an arbitrary (key, message) pair and return it."""
+ for key in self.iterkeys():
+ return (key, self.pop(key)) # This is only run once.
+ else:
+ raise KeyError('No messages in mailbox')
+
+ def update(self, arg=None):
+ """Change the messages that correspond to certain keys."""
+ if hasattr(arg, 'iteritems'):
+ source = arg.iteritems()
+ elif hasattr(arg, 'items'):
+ source = arg.items()
+ else:
+ source = arg
+ bad_key = False
+ for key, message in source:
+ try:
+ self[key] = message
+ except KeyError:
+ bad_key = True
+ if bad_key:
+ raise KeyError('No message with key(s)')
+
+ def flush(self):
+ """Write any pending changes to the disk."""
+ raise NotImplementedError('Method must be implemented by subclass')
+
+ def lock(self):
+ """Lock the mailbox."""
+ raise NotImplementedError('Method must be implemented by subclass')
+
+ def unlock(self):
+ """Unlock the mailbox if it is locked."""
+ raise NotImplementedError('Method must be implemented by subclass')
+
+ def close(self):
+ """Flush and close the mailbox."""
+ raise NotImplementedError('Method must be implemented by subclass')
+
+ def _dump_message(self, message, target, mangle_from_=False):
+ # Most files are opened in binary mode to allow predictable seeking.
+ # To get native line endings on disk, the user-friendly \n line endings
+ # used in strings and by email.Message are translated here.
+ """Dump message contents to target file."""
+ if isinstance(message, email.message.Message):
+ buffer = StringIO.StringIO()
+ gen = email.generator.Generator(buffer, mangle_from_, 0)
+ gen.flatten(message)
+ buffer.seek(0)
+ target.write(buffer.read().replace('\n', os.linesep))
+ elif isinstance(message, str):
+ if mangle_from_:
+ message = message.replace('\nFrom ', '\n>From ')
+ message = message.replace('\n', os.linesep)
+ target.write(message)
+ elif hasattr(message, 'read'):
+ while True:
+ line = message.readline()
+ if line == '':
+ break
+ if mangle_from_ and line.startswith('From '):
+ line = '>From ' + line[5:]
+ line = line.replace('\n', os.linesep)
+ target.write(line)
+ else:
+ raise TypeError('Invalid message type: %s' % type(message))
+
+
+class Maildir(Mailbox):
+ """A qmail-style Maildir mailbox."""
+
+ colon = ':'
+
+ def __init__(self, dirname, factory=rfc822.Message, create=True):
+ """Initialize a Maildir instance."""
+ Mailbox.__init__(self, dirname, factory, create)
+ self._paths = {
+ 'tmp': os.path.join(self._path, 'tmp'),
+ 'new': os.path.join(self._path, 'new'),
+ 'cur': os.path.join(self._path, 'cur'),
+ }
+ if not os.path.exists(self._path):
+ if create:
+ os.mkdir(self._path, 0700)
+ for path in self._paths.values():
+ os.mkdir(path, 0o700)
+ else:
+ raise NoSuchMailboxError(self._path)
+ self._toc = {}
+ self._toc_mtimes = {}
+ for subdir in ('cur', 'new'):
+ self._toc_mtimes[subdir] = os.path.getmtime(self._paths[subdir])
+ self._last_read = time.time() # Records last time we read cur/new
+ self._skewfactor = 0.1 # Adjust if os/fs clocks are skewing
+
+ def add(self, message):
+ """Add message and return assigned key."""
+ tmp_file = self._create_tmp()
+ try:
+ self._dump_message(message, tmp_file)
+ except BaseException:
+ tmp_file.close()
+ os.remove(tmp_file.name)
+ raise
+ _sync_close(tmp_file)
+ if isinstance(message, MaildirMessage):
+ subdir = message.get_subdir()
+ suffix = self.colon + message.get_info()
+ if suffix == self.colon:
+ suffix = ''
+ else:
+ subdir = 'new'
+ suffix = ''
+ uniq = os.path.basename(tmp_file.name).split(self.colon)[0]
+ dest = os.path.join(self._path, subdir, uniq + suffix)
+ try:
+ if hasattr(os, 'link'):
+ os.link(tmp_file.name, dest)
+ os.remove(tmp_file.name)
+ else:
+ os.rename(tmp_file.name, dest)
+ except OSError, e:
+ os.remove(tmp_file.name)
+ if e.errno == errno.EEXIST:
+ raise ExternalClashError('Name clash with existing message: %s'
+ % dest)
+ else:
+ raise
+ if isinstance(message, MaildirMessage):
+ os.utime(dest, (os.path.getatime(dest), message.get_date()))
+ return uniq
+
+ def remove(self, key):
+ """Remove the keyed message; raise KeyError if it doesn't exist."""
+ os.remove(os.path.join(self._path, self._lookup(key)))
+
+ def discard(self, key):
+ """If the keyed message exists, remove it."""
+ # This overrides an inapplicable implementation in the superclass.
+ try:
+ self.remove(key)
+ except KeyError:
+ pass
+ except OSError, e:
+ if e.errno != errno.ENOENT:
+ raise
+
+ def __setitem__(self, key, message):
+ """Replace the keyed message; raise KeyError if it doesn't exist."""
+ old_subpath = self._lookup(key)
+ temp_key = self.add(message)
+ temp_subpath = self._lookup(temp_key)
+ if isinstance(message, MaildirMessage):
+ # temp's subdir and suffix were specified by message.
+ dominant_subpath = temp_subpath
+ else:
+ # temp's subdir and suffix were defaults from add().
+ dominant_subpath = old_subpath
+ subdir = os.path.dirname(dominant_subpath)
+ if self.colon in dominant_subpath:
+ suffix = self.colon + dominant_subpath.split(self.colon)[-1]
+ else:
+ suffix = ''
+ self.discard(key)
+ new_path = os.path.join(self._path, subdir, key + suffix)
+ os.rename(os.path.join(self._path, temp_subpath), new_path)
+ if isinstance(message, MaildirMessage):
+ os.utime(new_path, (os.path.getatime(new_path),
+ message.get_date()))
+
+ def get_message(self, key):
+ """Return a Message representation or raise a KeyError."""
+ subpath = self._lookup(key)
+ f = open(os.path.join(self._path, subpath), 'r')
+ try:
+ if self._factory:
+ msg = self._factory(f)
+ else:
+ msg = MaildirMessage(f)
+ finally:
+ f.close()
+ subdir, name = os.path.split(subpath)
+ msg.set_subdir(subdir)
+ if self.colon in name:
+ msg.set_info(name.split(self.colon)[-1])
+ msg.set_date(os.path.getmtime(os.path.join(self._path, subpath)))
+ return msg
+
+ def get_string(self, key):
+ """Return a string representation or raise a KeyError."""
+ f = open(os.path.join(self._path, self._lookup(key)), 'r')
+ try:
+ return f.read()
+ finally:
+ f.close()
+
+ def get_file(self, key):
+ """Return a file-like representation or raise a KeyError."""
+ f = open(os.path.join(self._path, self._lookup(key)), 'rb')
+ return _ProxyFile(f)
+
+ def iterkeys(self):
+ """Return an iterator over keys."""
+ self._refresh()
+ for key in self._toc:
+ try:
+ self._lookup(key)
+ except KeyError:
+ continue
+ yield key
+
+ def has_key(self, key):
+ """Return True if the keyed message exists, False otherwise."""
+ self._refresh()
+ return key in self._toc
+
+ def __len__(self):
+ """Return a count of messages in the mailbox."""
+ self._refresh()
+ return len(self._toc)
+
+ def flush(self):
+ """Write any pending changes to disk."""
+ # Maildir changes are always written immediately, so there's nothing
+ # to do.
+ pass
+
+ def lock(self):
+ """Lock the mailbox."""
+ return
+
+ def unlock(self):
+ """Unlock the mailbox if it is locked."""
+ return
+
+ def close(self):
+ """Flush and close the mailbox."""
+ return
+
+ def list_folders(self):
+ """Return a list of folder names."""
+ result = []
+ for entry in os.listdir(self._path):
+ if len(entry) > 1 and entry[0] == '.' and \
+ os.path.isdir(os.path.join(self._path, entry)):
+ result.append(entry[1:])
+ return result
+
+ def get_folder(self, folder):
+ """Return a Maildir instance for the named folder."""
+ return Maildir(os.path.join(self._path, '.' + folder),
+ factory=self._factory,
+ create=False)
+
+ def add_folder(self, folder):
+ """Create a folder and return a Maildir instance representing it."""
+ path = os.path.join(self._path, '.' + folder)
+ result = Maildir(path, factory=self._factory)
+ maildirfolder_path = os.path.join(path, 'maildirfolder')
+ if not os.path.exists(maildirfolder_path):
+ os.close(os.open(maildirfolder_path, os.O_CREAT | os.O_WRONLY,
+ 0666))
+ return result
+
+ def remove_folder(self, folder):
+ """Delete the named folder, which must be empty."""
+ path = os.path.join(self._path, '.' + folder)
+ for entry in os.listdir(os.path.join(path, 'new')) + \
+ os.listdir(os.path.join(path, 'cur')):
+ if len(entry) < 1 or entry[0] != '.':
+ raise NotEmptyError('Folder contains message(s): %s' % folder)
+ for entry in os.listdir(path):
+ if entry != 'new' and entry != 'cur' and entry != 'tmp' and \
+ os.path.isdir(os.path.join(path, entry)):
+ raise NotEmptyError("Folder contains subdirectory '%s': %s" %
+ (folder, entry))
+ for root, dirs, files in os.walk(path, topdown=False):
+ for entry in files:
+ os.remove(os.path.join(root, entry))
+ for entry in dirs:
+ os.rmdir(os.path.join(root, entry))
+ os.rmdir(path)
+
+ def clean(self):
+ """Delete old files in "tmp"."""
+ now = time.time()
+ for entry in os.listdir(os.path.join(self._path, 'tmp')):
+ path = os.path.join(self._path, 'tmp', entry)
+ if now - os.path.getatime(path) > 129600: # 60 * 60 * 36
+ os.remove(path)
+
+ _count = 1 # This is used to generate unique file names.
+
+ def _create_tmp(self):
+ """Create a file in the tmp subdirectory and open and return it."""
+ now = time.time()
+ hostname = socket.gethostname()
+ if '/' in hostname:
+ hostname = hostname.replace('/', r'\057')
+ if ':' in hostname:
+ hostname = hostname.replace(':', r'\072')
+ uniq = "%s.M%sP%sQ%s.%s" % (int(now), int(now % 1 * 1e6), os.getpid(),
+ Maildir._count, hostname)
+ path = os.path.join(self._path, 'tmp', uniq)
+ try:
+ os.stat(path)
+ except OSError, e:
+ if e.errno == errno.ENOENT:
+ Maildir._count += 1
+ try:
+ return _create_carefully(path)
+ except OSError, e:
+ if e.errno != errno.EEXIST:
+ raise
+ else:
+ raise
+
+ # Fall through to here if stat succeeded or open raised EEXIST.
+ raise ExternalClashError('Name clash prevented file creation: %s' %
+ path)
+
+ def _refresh(self):
+ """Update table of contents mapping."""
+ # If it has been less than two seconds since the last _refresh() call,
+ # we have to unconditionally re-read the mailbox just in case it has
+ # been modified, because os.path.mtime() has a 2 sec resolution in the
+ # most common worst case (FAT) and a 1 sec resolution typically. This
+ # results in a few unnecessary re-reads when _refresh() is called
+ # multiple times in that interval, but once the clock ticks over, we
+ # will only re-read as needed. Because the filesystem might be being
+ # served by an independent system with its own clock, we record and
+ # compare with the mtimes from the filesystem. Because the other
+ # system's clock might be skewing relative to our clock, we add an
+ # extra delta to our wait. The default is one tenth second, but is an
+ # instance variable and so can be adjusted if dealing with a
+ # particularly skewed or irregular system.
+ if time.time() - self._last_read > 2 + self._skewfactor:
+ refresh = False
+ for subdir in self._toc_mtimes:
+ mtime = os.path.getmtime(self._paths[subdir])
+ if mtime > self._toc_mtimes[subdir]:
+ refresh = True
+ self._toc_mtimes[subdir] = mtime
+ if not refresh:
+ return
+ # Refresh toc
+ self._toc = {}
+ for subdir in self._toc_mtimes:
+ path = self._paths[subdir]
+ for entry in os.listdir(path):
+ p = os.path.join(path, entry)
+ if os.path.isdir(p):
+ continue
+ uniq = entry.split(self.colon)[0]
+ self._toc[uniq] = os.path.join(subdir, entry)
+ self._last_read = time.time()
+
+ def _lookup(self, key):
+ """Use TOC to return subpath for given key, or raise a KeyError."""
+ try:
+ if os.path.exists(os.path.join(self._path, self._toc[key])):
+ return self._toc[key]
+ except KeyError:
+ pass
+ self._refresh()
+ try:
+ return self._toc[key]
+ except KeyError:
+ raise KeyError('No message with key: %s' % key)
+
+ # This method is for backward compatibility only.
+ def next(self):
+ """Return the next message in a one-time iteration."""
+ if not hasattr(self, '_onetime_keys'):
+ self._onetime_keys = self.iterkeys()
+ while True:
+ try:
+ return self[self._onetime_keys.next()]
+ except StopIteration:
+ return None
+ except KeyError:
+ continue
+
+
+class _singlefileMailbox(Mailbox):
+ """A single-file mailbox."""
+
+ def __init__(self, path, factory=None, create=True):
+ """Initialize a single-file mailbox."""
+ Mailbox.__init__(self, path, factory, create)
+ try:
+ f = open(self._path, 'rb+')
+ except IOError, e:
+ if e.errno == errno.ENOENT:
+ if create:
+ f = open(self._path, 'wb+')
+ else:
+ raise NoSuchMailboxError(self._path)
+ elif e.errno in (errno.EACCES, errno.EROFS):
+ f = open(self._path, 'rb')
+ else:
+ raise
+ self._file = f
+ self._toc = None
+ self._next_key = 0
+ self._pending = False # No changes require rewriting the file.
+ self._locked = False
+ self._file_length = None # Used to record mailbox size
+
+ def add(self, message):
+ """Add message and return assigned key."""
+ self._lookup()
+ self._toc[self._next_key] = self._append_message(message)
+ self._next_key += 1
+ self._pending = True
+ return self._next_key - 1
+
+ def remove(self, key):
+ """Remove the keyed message; raise KeyError if it doesn't exist."""
+ self._lookup(key)
+ del self._toc[key]
+ self._pending = True
+
+ def __setitem__(self, key, message):
+ """Replace the keyed message; raise KeyError if it doesn't exist."""
+ self._lookup(key)
+ self._toc[key] = self._append_message(message)
+ self._pending = True
+
+ def iterkeys(self):
+ """Return an iterator over keys."""
+ self._lookup()
+ for key in self._toc.keys():
+ yield key
+
+ def has_key(self, key):
+ """Return True if the keyed message exists, False otherwise."""
+ self._lookup()
+ return key in self._toc
+
+ def __len__(self):
+ """Return a count of messages in the mailbox."""
+ self._lookup()
+ return len(self._toc)
+
+ def lock(self):
+ """Lock the mailbox."""
+ if not self._locked:
+ _lock_file(self._file)
+ self._locked = True
+
+ def unlock(self):
+ """Unlock the mailbox if it is locked."""
+ if self._locked:
+ _unlock_file(self._file)
+ self._locked = False
+
+ def flush(self):
+ """Write any pending changes to disk."""
+ if not self._pending:
+ return
+
+ # In order to be writing anything out at all, self._toc must
+ # already have been generated (and presumably has been modified
+ # by adding or deleting an item).
+ assert self._toc is not None
+
+ # Check length of self._file; if it's changed, some other process
+ # has modified the mailbox since we scanned it.
+ self._file.seek(0, 2)
+ cur_len = self._file.tell()
+ if cur_len != self._file_length:
+ raise ExternalClashError('Size of mailbox file changed '
+ '(expected %i, found %i)' %
+ (self._file_length, cur_len))
+
+ new_file = _create_temporary(self._path)
+ try:
+ new_toc = {}
+ self._pre_mailbox_hook(new_file)
+ for key in sorted(self._toc.keys()):
+ start, stop = self._toc[key]
+ self._file.seek(start)
+ self._pre_message_hook(new_file)
+ new_start = new_file.tell()
+ while True:
+ buffer = self._file.read(min(4096,
+ stop - self._file.tell()))
+ if buffer == '':
+ break
+ new_file.write(buffer)
+ new_toc[key] = (new_start, new_file.tell())
+ self._post_message_hook(new_file)
+ except:
+ new_file.close()
+ os.remove(new_file.name)
+ raise
+ _sync_close(new_file)
+ # self._file is about to get replaced, so no need to sync.
+ self._file.close()
+ try:
+ os.rename(new_file.name, self._path)
+ except OSError, e:
+ if e.errno == errno.EEXIST or \
+ (os.name == 'os2' and e.errno == errno.EACCES):
+ os.remove(self._path)
+ os.rename(new_file.name, self._path)
+ else:
+ raise
+ self._file = open(self._path, 'rb+')
+ self._toc = new_toc
+ self._pending = False
+ if self._locked:
+ _lock_file(self._file, dotlock=False)
+
+ def _pre_mailbox_hook(self, f):
+ """Called before writing the mailbox to file f."""
+ return
+
+ def _pre_message_hook(self, f):
+ """Called before writing each message to file f."""
+ return
+
+ def _post_message_hook(self, f):
+ """Called after writing each message to file f."""
+ return
+
+ def close(self):
+ """Flush and close the mailbox."""
+ self.flush()
+ if self._locked:
+ self.unlock()
+ self._file.close() # Sync has been done by self.flush() above.
+
+ def _lookup(self, key=None):
+ """Return (start, stop) or raise KeyError."""
+ if self._toc is None:
+ self._generate_toc()
+ if key is not None:
+ try:
+ return self._toc[key]
+ except KeyError:
+ raise KeyError('No message with key: %s' % key)
+
+ def _append_message(self, message):
+ """Append message to mailbox and return (start, stop) offsets."""
+ self._file.seek(0, 2)
+ before = self._file.tell()
+ try:
+ self._pre_message_hook(self._file)
+ offsets = self._install_message(message)
+ self._post_message_hook(self._file)
+ except BaseException:
+ self._file.truncate(before)
+ raise
+ self._file.flush()
+ self._file_length = self._file.tell() # Record current length of mailbox
+ return offsets
+
+
+
+class _mboxMMDF(_singlefileMailbox):
+ """An mbox or MMDF mailbox."""
+
+ _mangle_from_ = True
+
+ def get_message(self, key):
+ """Return a Message representation or raise a KeyError."""
+ start, stop = self._lookup(key)
+ self._file.seek(start)
+ from_line = self._file.readline().replace(os.linesep, '')
+ string = self._file.read(stop - self._file.tell())
+ msg = self._message_factory(string.replace(os.linesep, '\n'))
+ msg.set_from(from_line[5:])
+ return msg
+
+ def get_string(self, key, from_=False):
+ """Return a string representation or raise a KeyError."""
+ start, stop = self._lookup(key)
+ self._file.seek(start)
+ if not from_:
+ self._file.readline()
+ string = self._file.read(stop - self._file.tell())
+ return string.replace(os.linesep, '\n')
+
+ def get_file(self, key, from_=False):
+ """Return a file-like representation or raise a KeyError."""
+ start, stop = self._lookup(key)
+ self._file.seek(start)
+ if not from_:
+ self._file.readline()
+ return _PartialFile(self._file, self._file.tell(), stop)
+
+ def _install_message(self, message):
+ """Format a message and blindly write to self._file."""
+ from_line = None
+ if isinstance(message, str) and message.startswith('From '):
+ newline = message.find('\n')
+ if newline != -1:
+ from_line = message[:newline]
+ message = message[newline + 1:]
+ else:
+ from_line = message
+ message = ''
+ elif isinstance(message, _mboxMMDFMessage):
+ from_line = 'From ' + message.get_from()
+ elif isinstance(message, email.message.Message):
+ from_line = message.get_unixfrom() # May be None.
+ if from_line is None:
+ from_line = 'From MAILER-DAEMON %s' % time.asctime(time.gmtime())
+ start = self._file.tell()
+ self._file.write(from_line + os.linesep)
+ self._dump_message(message, self._file, self._mangle_from_)
+ stop = self._file.tell()
+ return (start, stop)
+
+
+class mbox(_mboxMMDF):
+ """A classic mbox mailbox."""
+
+ _mangle_from_ = True
+
+ def __init__(self, path, factory=None, create=True):
+ """Initialize an mbox mailbox."""
+ self._message_factory = mboxMessage
+ _mboxMMDF.__init__(self, path, factory, create)
+
+ def _pre_message_hook(self, f):
+ """Called before writing each message to file f."""
+ if f.tell() != 0:
+ f.write(os.linesep)
+
+ def _generate_toc(self):
+ """Generate key-to-(start, stop) table of contents."""
+ starts, stops = [], []
+ self._file.seek(0)
+ while True:
+ line_pos = self._file.tell()
+ line = self._file.readline()
+ if line.startswith('From '):
+ if len(stops) < len(starts):
+ stops.append(line_pos - len(os.linesep))
+ starts.append(line_pos)
+ elif line == '':
+ stops.append(line_pos)
+ break
+ self._toc = dict(enumerate(zip(starts, stops)))
+ self._next_key = len(self._toc)
+ self._file_length = self._file.tell()
+
+
+class MMDF(_mboxMMDF):
+ """An MMDF mailbox."""
+
+ def __init__(self, path, factory=None, create=True):
+ """Initialize an MMDF mailbox."""
+ self._message_factory = MMDFMessage
+ _mboxMMDF.__init__(self, path, factory, create)
+
+ def _pre_message_hook(self, f):
+ """Called before writing each message to file f."""
+ f.write('\001\001\001\001' + os.linesep)
+
+ def _post_message_hook(self, f):
+ """Called after writing each message to file f."""
+ f.write(os.linesep + '\001\001\001\001' + os.linesep)
+
+ def _generate_toc(self):
+ """Generate key-to-(start, stop) table of contents."""
+ starts, stops = [], []
+ self._file.seek(0)
+ next_pos = 0
+ while True:
+ line_pos = next_pos
+ line = self._file.readline()
+ next_pos = self._file.tell()
+ if line.startswith('\001\001\001\001' + os.linesep):
+ starts.append(next_pos)
+ while True:
+ line_pos = next_pos
+ line = self._file.readline()
+ next_pos = self._file.tell()
+ if line == '\001\001\001\001' + os.linesep:
+ stops.append(line_pos - len(os.linesep))
+ break
+ elif line == '':
+ stops.append(line_pos)
+ break
+ elif line == '':
+ break
+ self._toc = dict(enumerate(zip(starts, stops)))
+ self._next_key = len(self._toc)
+ self._file.seek(0, 2)
+ self._file_length = self._file.tell()
+
+
+class MH(Mailbox):
+ """An MH mailbox."""
+
+ def __init__(self, path, factory=None, create=True):
+ """Initialize an MH instance."""
+ Mailbox.__init__(self, path, factory, create)
+ if not os.path.exists(self._path):
+ if create:
+ os.mkdir(self._path, 0700)
+ os.close(os.open(os.path.join(self._path, '.mh_sequences'),
+ os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0600))
+ else:
+ raise NoSuchMailboxError(self._path)
+ self._locked = False
+
+ def add(self, message):
+ """Add message and return assigned key."""
+ keys = self.keys()
+ if len(keys) == 0:
+ new_key = 1
+ else:
+ new_key = max(keys) + 1
+ new_path = os.path.join(self._path, str(new_key))
+ f = _create_carefully(new_path)
+ closed = False
+ try:
+ if self._locked:
+ _lock_file(f)
+ try:
+ try:
+ self._dump_message(message, f)
+ except BaseException:
+ # Unlock and close so it can be deleted on Windows
+ if self._locked:
+ _unlock_file(f)
+ _sync_close(f)
+ closed = True
+ os.remove(new_path)
+ raise
+ if isinstance(message, MHMessage):
+ self._dump_sequences(message, new_key)
+ finally:
+ if self._locked:
+ _unlock_file(f)
+ finally:
+ if not closed:
+ _sync_close(f)
+ return new_key
+
+ def remove(self, key):
+ """Remove the keyed message; raise KeyError if it doesn't exist."""
+ path = os.path.join(self._path, str(key))
+ try:
+ f = open(path, 'rb+')
+ except IOError, e:
+ if e.errno == errno.ENOENT:
+ raise KeyError('No message with key: %s' % key)
+ else:
+ raise
+ else:
+ f.close()
+ os.remove(path)
+
+ def __setitem__(self, key, message):
+ """Replace the keyed message; raise KeyError if it doesn't exist."""
+ path = os.path.join(self._path, str(key))
+ try:
+ f = open(path, 'rb+')
+ except IOError, e:
+ if e.errno == errno.ENOENT:
+ raise KeyError('No message with key: %s' % key)
+ else:
+ raise
+ try:
+ if self._locked:
+ _lock_file(f)
+ try:
+ os.close(os.open(path, os.O_WRONLY | os.O_TRUNC))
+ self._dump_message(message, f)
+ if isinstance(message, MHMessage):
+ self._dump_sequences(message, key)
+ finally:
+ if self._locked:
+ _unlock_file(f)
+ finally:
+ _sync_close(f)
+
+ def get_message(self, key):
+ """Return a Message representation or raise a KeyError."""
+ try:
+ if self._locked:
+ f = open(os.path.join(self._path, str(key)), 'r+')
+ else:
+ f = open(os.path.join(self._path, str(key)), 'r')
+ except IOError, e:
+ if e.errno == errno.ENOENT:
+ raise KeyError('No message with key: %s' % key)
+ else:
+ raise
+ try:
+ if self._locked:
+ _lock_file(f)
+ try:
+ msg = MHMessage(f)
+ finally:
+ if self._locked:
+ _unlock_file(f)
+ finally:
+ f.close()
+ for name, key_list in self.get_sequences().iteritems():
+ if key in key_list:
+ msg.add_sequence(name)
+ return msg
+
+ def get_string(self, key):
+ """Return a string representation or raise a KeyError."""
+ try:
+ if self._locked:
+ f = open(os.path.join(self._path, str(key)), 'r+')
+ else:
+ f = open(os.path.join(self._path, str(key)), 'r')
+ except IOError, e:
+ if e.errno == errno.ENOENT:
+ raise KeyError('No message with key: %s' % key)
+ else:
+ raise
+ try:
+ if self._locked:
+ _lock_file(f)
+ try:
+ return f.read()
+ finally:
+ if self._locked:
+ _unlock_file(f)
+ finally:
+ f.close()
+
+ def get_file(self, key):
+ """Return a file-like representation or raise a KeyError."""
+ try:
+ f = open(os.path.join(self._path, str(key)), 'rb')
+ except IOError, e:
+ if e.errno == errno.ENOENT:
+ raise KeyError('No message with key: %s' % key)
+ else:
+ raise
+ return _ProxyFile(f)
+
+ def iterkeys(self):
+ """Return an iterator over keys."""
+ return iter(sorted(int(entry) for entry in os.listdir(self._path)
+ if entry.isdigit()))
+
+ def has_key(self, key):
+ """Return True if the keyed message exists, False otherwise."""
+ return os.path.exists(os.path.join(self._path, str(key)))
+
+ def __len__(self):
+ """Return a count of messages in the mailbox."""
+ return len(list(self.iterkeys()))
+
+ def lock(self):
+ """Lock the mailbox."""
+ if not self._locked:
+ self._file = open(os.path.join(self._path, '.mh_sequences'), 'rb+')
+ _lock_file(self._file)
+ self._locked = True
+
+ def unlock(self):
+ """Unlock the mailbox if it is locked."""
+ if self._locked:
+ _unlock_file(self._file)
+ _sync_close(self._file)
+ del self._file
+ self._locked = False
+
+ def flush(self):
+ """Write any pending changes to the disk."""
+ return
+
+ def close(self):
+ """Flush and close the mailbox."""
+ if self._locked:
+ self.unlock()
+
+ def list_folders(self):
+ """Return a list of folder names."""
+ result = []
+ for entry in os.listdir(self._path):
+ if os.path.isdir(os.path.join(self._path, entry)):
+ result.append(entry)
+ return result
+
+ def get_folder(self, folder):
+ """Return an MH instance for the named folder."""
+ return MH(os.path.join(self._path, folder),
+ factory=self._factory, create=False)
+
+ def add_folder(self, folder):
+ """Create a folder and return an MH instance representing it."""
+ return MH(os.path.join(self._path, folder),
+ factory=self._factory)
+
+ def remove_folder(self, folder):
+ """Delete the named folder, which must be empty."""
+ path = os.path.join(self._path, folder)
+ entries = os.listdir(path)
+ if entries == ['.mh_sequences']:
+ os.remove(os.path.join(path, '.mh_sequences'))
+ elif entries == []:
+ pass
+ else:
+ raise NotEmptyError('Folder not empty: %s' % self._path)
+ os.rmdir(path)
+
+ def get_sequences(self):
+ """Return a name-to-key-list dictionary to define each sequence."""
+ results = {}
+ f = open(os.path.join(self._path, '.mh_sequences'), 'r')
+ try:
+ all_keys = set(self.keys())
+ for line in f:
+ try:
+ name, contents = line.split(':')
+ keys = set()
+ for spec in contents.split():
+ if spec.isdigit():
+ keys.add(int(spec))
+ else:
+ start, stop = (int(x) for x in spec.split('-'))
+ keys.update(range(start, stop + 1))
+ results[name] = [key for key in sorted(keys) \
+ if key in all_keys]
+ if len(results[name]) == 0:
+ del results[name]
+ except ValueError:
+ raise FormatError('Invalid sequence specification: %s' %
+ line.rstrip())
+ finally:
+ f.close()
+ return results
+
+ def set_sequences(self, sequences):
+ """Set sequences using the given name-to-key-list dictionary."""
+ f = open(os.path.join(self._path, '.mh_sequences'), 'r+')
+ try:
+ os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC))
+ for name, keys in sequences.iteritems():
+ if len(keys) == 0:
+ continue
+ f.write('%s:' % name)
+ prev = None
+ completing = False
+ for key in sorted(set(keys)):
+ if key - 1 == prev:
+ if not completing:
+ completing = True
+ f.write('-')
+ elif completing:
+ completing = False
+ f.write('%s %s' % (prev, key))
+ else:
+ f.write(' %s' % key)
+ prev = key
+ if completing:
+ f.write(str(prev) + '\n')
+ else:
+ f.write('\n')
+ finally:
+ _sync_close(f)
+
+ def pack(self):
+ """Re-name messages to eliminate numbering gaps. Invalidates keys."""
+ sequences = self.get_sequences()
+ prev = 0
+ changes = []
+ for key in self.iterkeys():
+ if key - 1 != prev:
+ changes.append((key, prev + 1))
+ if hasattr(os, 'link'):
+ os.link(os.path.join(self._path, str(key)),
+ os.path.join(self._path, str(prev + 1)))
+ os.unlink(os.path.join(self._path, str(key)))
+ else:
+ os.rename(os.path.join(self._path, str(key)),
+ os.path.join(self._path, str(prev + 1)))
+ prev += 1
+ self._next_key = prev + 1
+ if len(changes) == 0:
+ return
+ for name, key_list in sequences.items():
+ for old, new in changes:
+ if old in key_list:
+ key_list[key_list.index(old)] = new
+ self.set_sequences(sequences)
+
+ def _dump_sequences(self, message, key):
+ """Inspect a new MHMessage and update sequences appropriately."""
+ pending_sequences = message.get_sequences()
+ all_sequences = self.get_sequences()
+ for name, key_list in all_sequences.iteritems():
+ if name in pending_sequences:
+ key_list.append(key)
+ elif key in key_list:
+ del key_list[key_list.index(key)]
+ for sequence in pending_sequences:
+ if sequence not in all_sequences:
+ all_sequences[sequence] = [key]
+ self.set_sequences(all_sequences)
+
+
+class Babyl(_singlefileMailbox):
+ """An Rmail-style Babyl mailbox."""
+
+ _special_labels = frozenset(('unseen', 'deleted', 'filed', 'answered',
+ 'forwarded', 'edited', 'resent'))
+
+ def __init__(self, path, factory=None, create=True):
+ """Initialize a Babyl mailbox."""
+ _singlefileMailbox.__init__(self, path, factory, create)
+ self._labels = {}
+
+ def add(self, message):
+ """Add message and return assigned key."""
+ key = _singlefileMailbox.add(self, message)
+ if isinstance(message, BabylMessage):
+ self._labels[key] = message.get_labels()
+ return key
+
+ def remove(self, key):
+ """Remove the keyed message; raise KeyError if it doesn't exist."""
+ _singlefileMailbox.remove(self, key)
+ if key in self._labels:
+ del self._labels[key]
+
+ def __setitem__(self, key, message):
+ """Replace the keyed message; raise KeyError if it doesn't exist."""
+ _singlefileMailbox.__setitem__(self, key, message)
+ if isinstance(message, BabylMessage):
+ self._labels[key] = message.get_labels()
+
+ def get_message(self, key):
+ """Return a Message representation or raise a KeyError."""
+ start, stop = self._lookup(key)
+ self._file.seek(start)
+ self._file.readline() # Skip '1,' line specifying labels.
+ original_headers = StringIO.StringIO()
+ while True:
+ line = self._file.readline()
+ if line == '*** EOOH ***' + os.linesep or line == '':
+ break
+ original_headers.write(line.replace(os.linesep, '\n'))
+ visible_headers = StringIO.StringIO()
+ while True:
+ line = self._file.readline()
+ if line == os.linesep or line == '':
+ break
+ visible_headers.write(line.replace(os.linesep, '\n'))
+ body = self._file.read(stop - self._file.tell()).replace(os.linesep,
+ '\n')
+ msg = BabylMessage(original_headers.getvalue() + body)
+ msg.set_visible(visible_headers.getvalue())
+ if key in self._labels:
+ msg.set_labels(self._labels[key])
+ return msg
+
+ def get_string(self, key):
+ """Return a string representation or raise a KeyError."""
+ start, stop = self._lookup(key)
+ self._file.seek(start)
+ self._file.readline() # Skip '1,' line specifying labels.
+ original_headers = StringIO.StringIO()
+ while True:
+ line = self._file.readline()
+ if line == '*** EOOH ***' + os.linesep or line == '':
+ break
+ original_headers.write(line.replace(os.linesep, '\n'))
+ while True:
+ line = self._file.readline()
+ if line == os.linesep or line == '':
+ break
+ return original_headers.getvalue() + \
+ self._file.read(stop - self._file.tell()).replace(os.linesep,
+ '\n')
+
+ def get_file(self, key):
+ """Return a file-like representation or raise a KeyError."""
+ return StringIO.StringIO(self.get_string(key).replace('\n',
+ os.linesep))
+
+ def get_labels(self):
+ """Return a list of user-defined labels in the mailbox."""
+ self._lookup()
+ labels = set()
+ for label_list in self._labels.values():
+ labels.update(label_list)
+ labels.difference_update(self._special_labels)
+ return list(labels)
+
+ def _generate_toc(self):
+ """Generate key-to-(start, stop) table of contents."""
+ starts, stops = [], []
+ self._file.seek(0)
+ next_pos = 0
+ label_lists = []
+ while True:
+ line_pos = next_pos
+ line = self._file.readline()
+ next_pos = self._file.tell()
+ if line == '\037\014' + os.linesep:
+ if len(stops) < len(starts):
+ stops.append(line_pos - len(os.linesep))
+ starts.append(next_pos)
+ labels = [label.strip() for label
+ in self._file.readline()[1:].split(',')
+ if label.strip() != '']
+ label_lists.append(labels)
+ elif line == '\037' or line == '\037' + os.linesep:
+ if len(stops) < len(starts):
+ stops.append(line_pos - len(os.linesep))
+ elif line == '':
+ stops.append(line_pos - len(os.linesep))
+ break
+ self._toc = dict(enumerate(zip(starts, stops)))
+ self._labels = dict(enumerate(label_lists))
+ self._next_key = len(self._toc)
+ self._file.seek(0, 2)
+ self._file_length = self._file.tell()
+
+ def _pre_mailbox_hook(self, f):
+ """Called before writing the mailbox to file f."""
+ f.write('BABYL OPTIONS:%sVersion: 5%sLabels:%s%s\037' %
+ (os.linesep, os.linesep, ','.join(self.get_labels()),
+ os.linesep))
+
+ def _pre_message_hook(self, f):
+ """Called before writing each message to file f."""
+ f.write('\014' + os.linesep)
+
+ def _post_message_hook(self, f):
+ """Called after writing each message to file f."""
+ f.write(os.linesep + '\037')
+
+ def _install_message(self, message):
+ """Write message contents and return (start, stop)."""
+ start = self._file.tell()
+ if isinstance(message, BabylMessage):
+ special_labels = []
+ labels = []
+ for label in message.get_labels():
+ if label in self._special_labels:
+ special_labels.append(label)
+ else:
+ labels.append(label)
+ self._file.write('1')
+ for label in special_labels:
+ self._file.write(', ' + label)
+ self._file.write(',,')
+ for label in labels:
+ self._file.write(' ' + label + ',')
+ self._file.write(os.linesep)
+ else:
+ self._file.write('1,,' + os.linesep)
+ if isinstance(message, email.message.Message):
+ orig_buffer = StringIO.StringIO()
+ orig_generator = email.generator.Generator(orig_buffer, False, 0)
+ orig_generator.flatten(message)
+ orig_buffer.seek(0)
+ while True:
+ line = orig_buffer.readline()
+ self._file.write(line.replace('\n', os.linesep))
+ if line == '\n' or line == '':
+ break
+ self._file.write('*** EOOH ***' + os.linesep)
+ if isinstance(message, BabylMessage):
+ vis_buffer = StringIO.StringIO()
+ vis_generator = email.generator.Generator(vis_buffer, False, 0)
+ vis_generator.flatten(message.get_visible())
+ while True:
+ line = vis_buffer.readline()
+ self._file.write(line.replace('\n', os.linesep))
+ if line == '\n' or line == '':
+ break
+ else:
+ orig_buffer.seek(0)
+ while True:
+ line = orig_buffer.readline()
+ self._file.write(line.replace('\n', os.linesep))
+ if line == '\n' or line == '':
+ break
+ while True:
+ buffer = orig_buffer.read(4096) # Buffer size is arbitrary.
+ if buffer == '':
+ break
+ self._file.write(buffer.replace('\n', os.linesep))
+ elif isinstance(message, str):
+ body_start = message.find('\n\n') + 2
+ if body_start - 2 != -1:
+ self._file.write(message[:body_start].replace('\n',
+ os.linesep))
+ self._file.write('*** EOOH ***' + os.linesep)
+ self._file.write(message[:body_start].replace('\n',
+ os.linesep))
+ self._file.write(message[body_start:].replace('\n',
+ os.linesep))
+ else:
+ self._file.write('*** EOOH ***' + os.linesep + os.linesep)
+ self._file.write(message.replace('\n', os.linesep))
+ elif hasattr(message, 'readline'):
+ original_pos = message.tell()
+ first_pass = True
+ while True:
+ line = message.readline()
+ self._file.write(line.replace('\n', os.linesep))
+ if line == '\n' or line == '':
+ self._file.write('*** EOOH ***' + os.linesep)
+ if first_pass:
+ first_pass = False
+ message.seek(original_pos)
+ else:
+ break
+ while True:
+ buffer = message.read(4096) # Buffer size is arbitrary.
+ if buffer == '':
+ break
+ self._file.write(buffer.replace('\n', os.linesep))
+ else:
+ raise TypeError('Invalid message type: %s' % type(message))
+ stop = self._file.tell()
+ return (start, stop)
+
+
+class Message(email.message.Message):
+ """Message with mailbox-format-specific properties."""
+
+ def __init__(self, message=None):
+ """Initialize a Message instance."""
+ if isinstance(message, email.message.Message):
+ self._become_message(copy.deepcopy(message))
+ if isinstance(message, Message):
+ message._explain_to(self)
+ elif isinstance(message, str):
+ self._become_message(email.message_from_string(message))
+ elif hasattr(message, "read"):
+ self._become_message(email.message_from_file(message))
+ elif message is None:
+ email.message.Message.__init__(self)
+ else:
+ raise TypeError('Invalid message type: %s' % type(message))
+
+ def _become_message(self, message):
+ """Assume the non-format-specific state of message."""
+ for name in ('_headers', '_unixfrom', '_payload', '_charset',
+ 'preamble', 'epilogue', 'defects', '_default_type'):
+ self.__dict__[name] = message.__dict__[name]
+
+ def _explain_to(self, message):
+ """Copy format-specific state to message insofar as possible."""
+ if isinstance(message, Message):
+ return # There's nothing format-specific to explain.
+ else:
+ raise TypeError('Cannot convert to specified type')
+
+
+class MaildirMessage(Message):
+ """Message with Maildir-specific properties."""
+
+ def __init__(self, message=None):
+ """Initialize a MaildirMessage instance."""
+ self._subdir = 'new'
+ self._info = ''
+ self._date = time.time()
+ Message.__init__(self, message)
+
+ def get_subdir(self):
+ """Return 'new' or 'cur'."""
+ return self._subdir
+
+ def set_subdir(self, subdir):
+ """Set subdir to 'new' or 'cur'."""
+ if subdir == 'new' or subdir == 'cur':
+ self._subdir = subdir
+ else:
+ raise ValueError("subdir must be 'new' or 'cur': %s" % subdir)
+
+ def get_flags(self):
+ """Return as a string the flags that are set."""
+ if self._info.startswith('2,'):
+ return self._info[2:]
+ else:
+ return ''
+
+ def set_flags(self, flags):
+ """Set the given flags and unset all others."""
+ self._info = '2,' + ''.join(sorted(flags))
+
+ def add_flag(self, flag):
+ """Set the given flag(s) without changing others."""
+ self.set_flags(''.join(set(self.get_flags()) | set(flag)))
+
+ def remove_flag(self, flag):
+ """Unset the given string flag(s) without changing others."""
+ if self.get_flags() != '':
+ self.set_flags(''.join(set(self.get_flags()) - set(flag)))
+
+ def get_date(self):
+ """Return delivery date of message, in seconds since the epoch."""
+ return self._date
+
+ def set_date(self, date):
+ """Set delivery date of message, in seconds since the epoch."""
+ try:
+ self._date = float(date)
+ except ValueError:
+ raise TypeError("can't convert to float: %s" % date)
+
+ def get_info(self):
+ """Get the message's "info" as a string."""
+ return self._info
+
+ def set_info(self, info):
+ """Set the message's "info" string."""
+ if isinstance(info, str):
+ self._info = info
+ else:
+ raise TypeError('info must be a string: %s' % type(info))
+
+ def _explain_to(self, message):
+ """Copy Maildir-specific state to message insofar as possible."""
+ if isinstance(message, MaildirMessage):
+ message.set_flags(self.get_flags())
+ message.set_subdir(self.get_subdir())
+ message.set_date(self.get_date())
+ elif isinstance(message, _mboxMMDFMessage):
+ flags = set(self.get_flags())
+ if 'S' in flags:
+ message.add_flag('R')
+ if self.get_subdir() == 'cur':
+ message.add_flag('O')
+ if 'T' in flags:
+ message.add_flag('D')
+ if 'F' in flags:
+ message.add_flag('F')
+ if 'R' in flags:
+ message.add_flag('A')
+ message.set_from('MAILER-DAEMON', time.gmtime(self.get_date()))
+ elif isinstance(message, MHMessage):
+ flags = set(self.get_flags())
+ if 'S' not in flags:
+ message.add_sequence('unseen')
+ if 'R' in flags:
+ message.add_sequence('replied')
+ if 'F' in flags:
+ message.add_sequence('flagged')
+ elif isinstance(message, BabylMessage):
+ flags = set(self.get_flags())
+ if 'S' not in flags:
+ message.add_label('unseen')
+ if 'T' in flags:
+ message.add_label('deleted')
+ if 'R' in flags:
+ message.add_label('answered')
+ if 'P' in flags:
+ message.add_label('forwarded')
+ elif isinstance(message, Message):
+ pass
+ else:
+ raise TypeError('Cannot convert to specified type: %s' %
+ type(message))
+
+
+class _mboxMMDFMessage(Message):
+ """Message with mbox- or MMDF-specific properties."""
+
+ def __init__(self, message=None):
+ """Initialize an mboxMMDFMessage instance."""
+ self.set_from('MAILER-DAEMON', True)
+ if isinstance(message, email.message.Message):
+ unixfrom = message.get_unixfrom()
+ if unixfrom is not None and unixfrom.startswith('From '):
+ self.set_from(unixfrom[5:])
+ Message.__init__(self, message)
+
+ def get_from(self):
+ """Return contents of "From " line."""
+ return self._from
+
+ def set_from(self, from_, time_=None):
+ """Set "From " line, formatting and appending time_ if specified."""
+ if time_ is not None:
+ if time_ is True:
+ time_ = time.gmtime()
+ from_ += ' ' + time.asctime(time_)
+ self._from = from_
+
+ def get_flags(self):
+ """Return as a string the flags that are set."""
+ return self.get('Status', '') + self.get('X-Status', '')
+
+ def set_flags(self, flags):
+ """Set the given flags and unset all others."""
+ flags = set(flags)
+ status_flags, xstatus_flags = '', ''
+ for flag in ('R', 'O'):
+ if flag in flags:
+ status_flags += flag
+ flags.remove(flag)
+ for flag in ('D', 'F', 'A'):
+ if flag in flags:
+ xstatus_flags += flag
+ flags.remove(flag)
+ xstatus_flags += ''.join(sorted(flags))
+ try:
+ self.replace_header('Status', status_flags)
+ except KeyError:
+ self.add_header('Status', status_flags)
+ try:
+ self.replace_header('X-Status', xstatus_flags)
+ except KeyError:
+ self.add_header('X-Status', xstatus_flags)
+
+ def add_flag(self, flag):
+ """Set the given flag(s) without changing others."""
+ self.set_flags(''.join(set(self.get_flags()) | set(flag)))
+
+ def remove_flag(self, flag):
+ """Unset the given string flag(s) without changing others."""
+ if 'Status' in self or 'X-Status' in self:
+ self.set_flags(''.join(set(self.get_flags()) - set(flag)))
+
+ def _explain_to(self, message):
+ """Copy mbox- or MMDF-specific state to message insofar as possible."""
+ if isinstance(message, MaildirMessage):
+ flags = set(self.get_flags())
+ if 'O' in flags:
+ message.set_subdir('cur')
+ if 'F' in flags:
+ message.add_flag('F')
+ if 'A' in flags:
+ message.add_flag('R')
+ if 'R' in flags:
+ message.add_flag('S')
+ if 'D' in flags:
+ message.add_flag('T')
+ del message['status']
+ del message['x-status']
+ maybe_date = ' '.join(self.get_from().split()[-5:])
+ try:
+ message.set_date(calendar.timegm(time.strptime(maybe_date,
+ '%a %b %d %H:%M:%S %Y')))
+ except (ValueError, OverflowError):
+ pass
+ elif isinstance(message, _mboxMMDFMessage):
+ message.set_flags(self.get_flags())
+ message.set_from(self.get_from())
+ elif isinstance(message, MHMessage):
+ flags = set(self.get_flags())
+ if 'R' not in flags:
+ message.add_sequence('unseen')
+ if 'A' in flags:
+ message.add_sequence('replied')
+ if 'F' in flags:
+ message.add_sequence('flagged')
+ del message['status']
+ del message['x-status']
+ elif isinstance(message, BabylMessage):
+ flags = set(self.get_flags())
+ if 'R' not in flags:
+ message.add_label('unseen')
+ if 'D' in flags:
+ message.add_label('deleted')
+ if 'A' in flags:
+ message.add_label('answered')
+ del message['status']
+ del message['x-status']
+ elif isinstance(message, Message):
+ pass
+ else:
+ raise TypeError('Cannot convert to specified type: %s' %
+ type(message))
+
+
+class mboxMessage(_mboxMMDFMessage):
+ """Message with mbox-specific properties."""
+
+
+class MHMessage(Message):
+ """Message with MH-specific properties."""
+
+ def __init__(self, message=None):
+ """Initialize an MHMessage instance."""
+ self._sequences = []
+ Message.__init__(self, message)
+
+ def get_sequences(self):
+ """Return a list of sequences that include the message."""
+ return self._sequences[:]
+
+ def set_sequences(self, sequences):
+ """Set the list of sequences that include the message."""
+ self._sequences = list(sequences)
+
+ def add_sequence(self, sequence):
+ """Add sequence to list of sequences including the message."""
+ if isinstance(sequence, str):
+ if not sequence in self._sequences:
+ self._sequences.append(sequence)
+ else:
+ raise TypeError('sequence must be a string: %s' % type(sequence))
+
+ def remove_sequence(self, sequence):
+ """Remove sequence from the list of sequences including the message."""
+ try:
+ self._sequences.remove(sequence)
+ except ValueError:
+ pass
+
+ def _explain_to(self, message):
+ """Copy MH-specific state to message insofar as possible."""
+ if isinstance(message, MaildirMessage):
+ sequences = set(self.get_sequences())
+ if 'unseen' in sequences:
+ message.set_subdir('cur')
+ else:
+ message.set_subdir('cur')
+ message.add_flag('S')
+ if 'flagged' in sequences:
+ message.add_flag('F')
+ if 'replied' in sequences:
+ message.add_flag('R')
+ elif isinstance(message, _mboxMMDFMessage):
+ sequences = set(self.get_sequences())
+ if 'unseen' not in sequences:
+ message.add_flag('RO')
+ else:
+ message.add_flag('O')
+ if 'flagged' in sequences:
+ message.add_flag('F')
+ if 'replied' in sequences:
+ message.add_flag('A')
+ elif isinstance(message, MHMessage):
+ for sequence in self.get_sequences():
+ message.add_sequence(sequence)
+ elif isinstance(message, BabylMessage):
+ sequences = set(self.get_sequences())
+ if 'unseen' in sequences:
+ message.add_label('unseen')
+ if 'replied' in sequences:
+ message.add_label('answered')
+ elif isinstance(message, Message):
+ pass
+ else:
+ raise TypeError('Cannot convert to specified type: %s' %
+ type(message))
+
+
+class BabylMessage(Message):
+ """Message with Babyl-specific properties."""
+
+ def __init__(self, message=None):
+ """Initialize an BabylMessage instance."""
+ self._labels = []
+ self._visible = Message()
+ Message.__init__(self, message)
+
+ def get_labels(self):
+ """Return a list of labels on the message."""
+ return self._labels[:]
+
+ def set_labels(self, labels):
+ """Set the list of labels on the message."""
+ self._labels = list(labels)
+
+ def add_label(self, label):
+ """Add label to list of labels on the message."""
+ if isinstance(label, str):
+ if label not in self._labels:
+ self._labels.append(label)
+ else:
+ raise TypeError('label must be a string: %s' % type(label))
+
+ def remove_label(self, label):
+ """Remove label from the list of labels on the message."""
+ try:
+ self._labels.remove(label)
+ except ValueError:
+ pass
+
+ def get_visible(self):
+ """Return a Message representation of visible headers."""
+ return Message(self._visible)
+
+ def set_visible(self, visible):
+ """Set the Message representation of visible headers."""
+ self._visible = Message(visible)
+
+ def update_visible(self):
+ """Update and/or sensibly generate a set of visible headers."""
+ for header in self._visible.keys():
+ if header in self:
+ self._visible.replace_header(header, self[header])
+ else:
+ del self._visible[header]
+ for header in ('Date', 'From', 'Reply-To', 'To', 'CC', 'Subject'):
+ if header in self and header not in self._visible:
+ self._visible[header] = self[header]
+
+ def _explain_to(self, message):
+ """Copy Babyl-specific state to message insofar as possible."""
+ if isinstance(message, MaildirMessage):
+ labels = set(self.get_labels())
+ if 'unseen' in labels:
+ message.set_subdir('cur')
+ else:
+ message.set_subdir('cur')
+ message.add_flag('S')
+ if 'forwarded' in labels or 'resent' in labels:
+ message.add_flag('P')
+ if 'answered' in labels:
+ message.add_flag('R')
+ if 'deleted' in labels:
+ message.add_flag('T')
+ elif isinstance(message, _mboxMMDFMessage):
+ labels = set(self.get_labels())
+ if 'unseen' not in labels:
+ message.add_flag('RO')
+ else:
+ message.add_flag('O')
+ if 'deleted' in labels:
+ message.add_flag('D')
+ if 'answered' in labels:
+ message.add_flag('A')
+ elif isinstance(message, MHMessage):
+ labels = set(self.get_labels())
+ if 'unseen' in labels:
+ message.add_sequence('unseen')
+ if 'answered' in labels:
+ message.add_sequence('replied')
+ elif isinstance(message, BabylMessage):
+ message.set_visible(self.get_visible())
+ for label in self.get_labels():
+ message.add_label(label)
+ elif isinstance(message, Message):
+ pass
+ else:
+ raise TypeError('Cannot convert to specified type: %s' %
+ type(message))
+
+
+class MMDFMessage(_mboxMMDFMessage):
+ """Message with MMDF-specific properties."""
+
+
+class _ProxyFile:
+ """A read-only wrapper of a file."""
+
+ def __init__(self, f, pos=None):
+ """Initialize a _ProxyFile."""
+ self._file = f
+ if pos is None:
+ self._pos = f.tell()
+ else:
+ self._pos = pos
+
+ def read(self, size=None):
+ """Read bytes."""
+ return self._read(size, self._file.read)
+
+ def readline(self, size=None):
+ """Read a line."""
+ return self._read(size, self._file.readline)
+
+ def readlines(self, sizehint=None):
+ """Read multiple lines."""
+ result = []
+ for line in self:
+ result.append(line)
+ if sizehint is not None:
+ sizehint -= len(line)
+ if sizehint <= 0:
+ break
+ return result
+
+ def __iter__(self):
+ """Iterate over lines."""
+ return iter(self.readline, "")
+
+ def tell(self):
+ """Return the position."""
+ return self._pos
+
+ def seek(self, offset, whence=0):
+ """Change position."""
+ if whence == 1:
+ self._file.seek(self._pos)
+ self._file.seek(offset, whence)
+ self._pos = self._file.tell()
+
+ def close(self):
+ """Close the file."""
+ del self._file
+
+ def _read(self, size, read_method):
+ """Read size bytes using read_method."""
+ if size is None:
+ size = -1
+ self._file.seek(self._pos)
+ result = read_method(size)
+ self._pos = self._file.tell()
+ return result
+
+
+class _PartialFile(_ProxyFile):
+ """A read-only wrapper of part of a file."""
+
+ def __init__(self, f, start=None, stop=None):
+ """Initialize a _PartialFile."""
+ _ProxyFile.__init__(self, f, start)
+ self._start = start
+ self._stop = stop
+
+ def tell(self):
+ """Return the position with respect to start."""
+ return _ProxyFile.tell(self) - self._start
+
+ def seek(self, offset, whence=0):
+ """Change position, possibly with respect to start or stop."""
+ if whence == 0:
+ self._pos = self._start
+ whence = 1
+ elif whence == 2:
+ self._pos = self._stop
+ whence = 1
+ _ProxyFile.seek(self, offset, whence)
+
+ def _read(self, size, read_method):
+ """Read size bytes using read_method, honoring start and stop."""
+ remaining = self._stop - self._pos
+ if remaining <= 0:
+ return ''
+ if size is None or size < 0 or size > remaining:
+ size = remaining
+ return _ProxyFile._read(self, size, read_method)
+
+
+def _lock_file(f, dotlock=True):
+ """Lock file f using lockf and dot locking."""
+ dotlock_done = False
+ try:
+ if fcntl:
+ try:
+ fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ except IOError, e:
+ if e.errno in (errno.EAGAIN, errno.EACCES, errno.EROFS):
+ raise ExternalClashError('lockf: lock unavailable: %s' %
+ f.name)
+ else:
+ raise
+ if dotlock:
+ try:
+ pre_lock = _create_temporary(f.name + '.lock')
+ pre_lock.close()
+ except IOError, e:
+ if e.errno in (errno.EACCES, errno.EROFS):
+ return # Without write access, just skip dotlocking.
+ else:
+ raise
+ try:
+ if hasattr(os, 'link'):
+ os.link(pre_lock.name, f.name + '.lock')
+ dotlock_done = True
+ os.unlink(pre_lock.name)
+ else:
+ os.rename(pre_lock.name, f.name + '.lock')
+ dotlock_done = True
+ except OSError, e:
+ if e.errno == errno.EEXIST or \
+ (os.name == 'os2' and e.errno == errno.EACCES):
+ os.remove(pre_lock.name)
+ raise ExternalClashError('dot lock unavailable: %s' %
+ f.name)
+ else:
+ raise
+ except:
+ if fcntl:
+ fcntl.lockf(f, fcntl.LOCK_UN)
+ if dotlock_done:
+ os.remove(f.name + '.lock')
+ raise
+
+def _unlock_file(f):
+ """Unlock file f using lockf and dot locking."""
+ if fcntl:
+ fcntl.lockf(f, fcntl.LOCK_UN)
+ if os.path.exists(f.name + '.lock'):
+ os.remove(f.name + '.lock')
+
+def _create_carefully(path):
+ """Create a file if it doesn't exist and open for reading and writing."""
+ fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0666)
+ try:
+ return open(path, 'rb+')
+ finally:
+ os.close(fd)
+
+def _create_temporary(path):
+ """Create a temp file based on path and open for reading and writing."""
+ return _create_carefully('%s.%s.%s.%s' % (path, int(time.time()),
+ socket.gethostname(),
+ os.getpid()))
+
+def _sync_flush(f):
+ """Ensure changes to file f are physically on disk."""
+ f.flush()
+ if hasattr(os, 'fsync'):
+ os.fsync(f.fileno())
+
+def _sync_close(f):
+ """Close file f, ensuring all changes are physically on disk."""
+ _sync_flush(f)
+ f.close()
+
+## Start: classes from the original module (for backward compatibility).
+
+# Note that the Maildir class, whose name is unchanged, itself offers a next()
+# method for backward compatibility.
+
+class _Mailbox:
+
+ def __init__(self, fp, factory=rfc822.Message):
+ self.fp = fp
+ self.seekp = 0
+ self.factory = factory
+
+ def __iter__(self):
+ return iter(self.next, None)
+
+ def next(self):
+ while 1:
+ self.fp.seek(self.seekp)
+ try:
+ self._search_start()
+ except EOFError:
+ self.seekp = self.fp.tell()
+ return None
+ start = self.fp.tell()
+ self._search_end()
+ self.seekp = stop = self.fp.tell()
+ if start != stop:
+ break
+ return self.factory(_PartialFile(self.fp, start, stop))
+
+# Recommended to use PortableUnixMailbox instead!
+class UnixMailbox(_Mailbox):
+
+ def _search_start(self):
+ while 1:
+ pos = self.fp.tell()
+ line = self.fp.readline()
+ if not line:
+ raise EOFError
+ if line[:5] == 'From ' and self._isrealfromline(line):
+ self.fp.seek(pos)
+ return
+
+ def _search_end(self):
+ self.fp.readline() # Throw away header line
+ while 1:
+ pos = self.fp.tell()
+ line = self.fp.readline()
+ if not line:
+ return
+ if line[:5] == 'From ' and self._isrealfromline(line):
+ self.fp.seek(pos)
+ return
+
+ # An overridable mechanism to test for From-line-ness. You can either
+ # specify a different regular expression or define a whole new
+ # _isrealfromline() method. Note that this only gets called for lines
+ # starting with the 5 characters "From ".
+ #
+ # BAW: According to
+ #http://home.netscape.com/eng/mozilla/2.0/relnotes/demo/content-length.html
+ # the only portable, reliable way to find message delimiters in a BSD (i.e
+ # Unix mailbox) style folder is to search for "\n\nFrom .*\n", or at the
+ # beginning of the file, "^From .*\n". While _fromlinepattern below seems
+ # like a good idea, in practice, there are too many variations for more
+ # strict parsing of the line to be completely accurate.
+ #
+ # _strict_isrealfromline() is the old version which tries to do stricter
+ # parsing of the From_ line. _portable_isrealfromline() simply returns
+ # true, since it's never called if the line doesn't already start with
+ # "From ".
+ #
+ # This algorithm, and the way it interacts with _search_start() and
+ # _search_end() may not be completely correct, because it doesn't check
+ # that the two characters preceding "From " are \n\n or the beginning of
+ # the file. Fixing this would require a more extensive rewrite than is
+ # necessary. For convenience, we've added a PortableUnixMailbox class
+ # which does no checking of the format of the 'From' line.
+
+ _fromlinepattern = (r"From \s*[^\s]+\s+\w\w\w\s+\w\w\w\s+\d?\d\s+"
+ r"\d?\d:\d\d(:\d\d)?(\s+[^\s]+)?\s+\d\d\d\d\s*"
+ r"[^\s]*\s*"
+ "$")
+ _regexp = None
+
+ def _strict_isrealfromline(self, line):
+ if not self._regexp:
+ import re
+ self._regexp = re.compile(self._fromlinepattern)
+ return self._regexp.match(line)
+
+ def _portable_isrealfromline(self, line):
+ return True
+
+ _isrealfromline = _strict_isrealfromline
+
+
+class PortableUnixMailbox(UnixMailbox):
+ _isrealfromline = UnixMailbox._portable_isrealfromline
+
+
+class MmdfMailbox(_Mailbox):
+
+ def _search_start(self):
+ while 1:
+ line = self.fp.readline()
+ if not line:
+ raise EOFError
+ if line[:5] == '\001\001\001\001\n':
+ return
+
+ def _search_end(self):
+ while 1:
+ pos = self.fp.tell()
+ line = self.fp.readline()
+ if not line:
+ return
+ if line == '\001\001\001\001\n':
+ self.fp.seek(pos)
+ return
+
+
+class MHMailbox:
+
+ def __init__(self, dirname, factory=rfc822.Message):
+ import re
+ pat = re.compile('^[1-9][0-9]*$')
+ self.dirname = dirname
+ # the three following lines could be combined into:
+ # list = map(long, filter(pat.match, os.listdir(self.dirname)))
+ list = os.listdir(self.dirname)
+ list = filter(pat.match, list)
+ list = map(long, list)
+ list.sort()
+ # This only works in Python 1.6 or later;
+ # before that str() added 'L':
+ self.boxes = map(str, list)
+ self.boxes.reverse()
+ self.factory = factory
+
+ def __iter__(self):
+ return iter(self.next, None)
+
+ def next(self):
+ if not self.boxes:
+ return None
+ fn = self.boxes.pop()
+ fp = open(os.path.join(self.dirname, fn))
+ msg = self.factory(fp)
+ try:
+ msg._mh_msgno = fn
+ except (AttributeError, TypeError):
+ pass
+ return msg
+
+
+class BabylMailbox(_Mailbox):
+
+ def _search_start(self):
+ while 1:
+ line = self.fp.readline()
+ if not line:
+ raise EOFError
+ if line == '*** EOOH ***\n':
+ return
+
+ def _search_end(self):
+ while 1:
+ pos = self.fp.tell()
+ line = self.fp.readline()
+ if not line:
+ return
+ if line == '\037\014\n' or line == '\037':
+ self.fp.seek(pos)
+ return
+
+## End: classes from the original module (for backward compatibility).
+
+
+class Error(Exception):
+ """Raised for module-specific errors."""
+
+class NoSuchMailboxError(Error):
+ """The specified mailbox does not exist and won't be created."""
+
+class NotEmptyError(Error):
+ """The specified mailbox is not empty and deletion was requested."""
+
+class ExternalClashError(Error):
+ """Another process caused an action to fail."""
+
+class FormatError(Error):
+ """A file appears to have an invalid format."""
diff --git a/lib-python/2.7/test/test_mailbox.py b/lib-python/2.7/test/test_mailbox.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/test_mailbox.py
@@ -0,0 +1,1970 @@
+import os
+import sys
+import time
+import stat
+import socket
+import email
+import email.message
+import re
+import StringIO
+from test import test_support
+import unittest
+import mailbox
+import glob
+try:
+ import fcntl
+except ImportError:
+ pass
+
+# Silence Py3k warning
+rfc822 = test_support.import_module('rfc822', deprecated=True)
+
+class TestBase(unittest.TestCase):
+
+ def _check_sample(self, msg):
+ # Inspect a mailbox.Message representation of the sample message
+ self.assertIsInstance(msg, email.message.Message)
+ self.assertIsInstance(msg, mailbox.Message)
+ for key, value in _sample_headers.iteritems():
+ self.assertIn(value, msg.get_all(key))
+ self.assertTrue(msg.is_multipart())
+ self.assertEqual(len(msg.get_payload()), len(_sample_payloads))
+ for i, payload in enumerate(_sample_payloads):
+ part = msg.get_payload(i)
+ self.assertIsInstance(part, email.message.Message)
+ self.assertNotIsInstance(part, mailbox.Message)
+ self.assertEqual(part.get_payload(), payload)
+
+ def _delete_recursively(self, target):
+ # Delete a file or delete a directory recursively
+ if os.path.isdir(target):
+ for path, dirs, files in os.walk(target, topdown=False):
+ for name in files:
+ os.remove(os.path.join(path, name))
+ for name in dirs:
+ os.rmdir(os.path.join(path, name))
+ os.rmdir(target)
+ elif os.path.exists(target):
+ os.remove(target)
+
+
+class TestMailbox(TestBase):
+
+ _factory = None # Overridden by subclasses to reuse tests
+ _template = 'From: foo\n\n%s'
+
+ def setUp(self):
+ self._path = test_support.TESTFN
+ self._delete_recursively(self._path)
+ self._box = self._factory(self._path)
+
+ def tearDown(self):
+ self._box.close()
+ self._delete_recursively(self._path)
+
+ def test_add(self):
+ # Add copies of a sample message
+ keys = []
+ keys.append(self._box.add(self._template % 0))
+ self.assertEqual(len(self._box), 1)
+ keys.append(self._box.add(mailbox.Message(_sample_message)))
+ self.assertEqual(len(self._box), 2)
+ keys.append(self._box.add(email.message_from_string(_sample_message)))
+ self.assertEqual(len(self._box), 3)
+ keys.append(self._box.add(StringIO.StringIO(_sample_message)))
+ self.assertEqual(len(self._box), 4)
+ keys.append(self._box.add(_sample_message))
+ self.assertEqual(len(self._box), 5)
+ self.assertEqual(self._box.get_string(keys[0]), self._template % 0)
+ for i in (1, 2, 3, 4):
+ self._check_sample(self._box[keys[i]])
+
+ def test_remove(self):
+ # Remove messages using remove()
+ self._test_remove_or_delitem(self._box.remove)
+
+ def test_delitem(self):
+ # Remove messages using __delitem__()
+ self._test_remove_or_delitem(self._box.__delitem__)
+
+ def _test_remove_or_delitem(self, method):
+ # (Used by test_remove() and test_delitem().)
+ key0 = self._box.add(self._template % 0)
+ key1 = self._box.add(self._template % 1)
+ self.assertEqual(len(self._box), 2)
+ method(key0)
+ l = len(self._box)
+ self.assertEqual(l, 1)
+ self.assertRaises(KeyError, lambda: self._box[key0])
+ self.assertRaises(KeyError, lambda: method(key0))
+ self.assertEqual(self._box.get_string(key1), self._template % 1)
+ key2 = self._box.add(self._template % 2)
+ self.assertEqual(len(self._box), 2)
+ method(key2)
+ l = len(self._box)
+ self.assertEqual(l, 1)
+ self.assertRaises(KeyError, lambda: self._box[key2])
+ self.assertRaises(KeyError, lambda: method(key2))
+ self.assertEqual(self._box.get_string(key1), self._template % 1)
+ method(key1)
+ self.assertEqual(len(self._box), 0)
+ self.assertRaises(KeyError, lambda: self._box[key1])
+ self.assertRaises(KeyError, lambda: method(key1))
+
+ def test_discard(self, repetitions=10):
+ # Discard messages
+ key0 = self._box.add(self._template % 0)
+ key1 = self._box.add(self._template % 1)
+ self.assertEqual(len(self._box), 2)
+ self._box.discard(key0)
+ self.assertEqual(len(self._box), 1)
+ self.assertRaises(KeyError, lambda: self._box[key0])
+ self._box.discard(key0)
+ self.assertEqual(len(self._box), 1)
+ self.assertRaises(KeyError, lambda: self._box[key0])
+
+ def test_get(self):
+ # Retrieve messages using get()
+ key0 = self._box.add(self._template % 0)
+ msg = self._box.get(key0)
+ self.assertEqual(msg['from'], 'foo')
+ self.assertEqual(msg.get_payload(), '0')
+ self.assertIs(self._box.get('foo'), None)
+ self.assertFalse(self._box.get('foo', False))
+ self._box.close()
+ self._box = self._factory(self._path, factory=rfc822.Message)
+ key1 = self._box.add(self._template % 1)
+ msg = self._box.get(key1)
+ self.assertEqual(msg['from'], 'foo')
+ self.assertEqual(msg.fp.read(), '1')
+
+ def test_getitem(self):
+ # Retrieve message using __getitem__()
+ key0 = self._box.add(self._template % 0)
+ msg = self._box[key0]
+ self.assertEqual(msg['from'], 'foo')
+ self.assertEqual(msg.get_payload(), '0')
+ self.assertRaises(KeyError, lambda: self._box['foo'])
+ self._box.discard(key0)
+ self.assertRaises(KeyError, lambda: self._box[key0])
+
+ def test_get_message(self):
+ # Get Message representations of messages
+ key0 = self._box.add(self._template % 0)
+ key1 = self._box.add(_sample_message)
+ msg0 = self._box.get_message(key0)
+ self.assertIsInstance(msg0, mailbox.Message)
+ self.assertEqual(msg0['from'], 'foo')
+ self.assertEqual(msg0.get_payload(), '0')
+ self._check_sample(self._box.get_message(key1))
+
+ def test_get_string(self):
+ # Get string representations of messages
+ key0 = self._box.add(self._template % 0)
+ key1 = self._box.add(_sample_message)
+ self.assertEqual(self._box.get_string(key0), self._template % 0)
+ self.assertEqual(self._box.get_string(key1), _sample_message)
+
+ def test_get_file(self):
+ # Get file representations of messages
+ key0 = self._box.add(self._template % 0)
+ key1 = self._box.add(_sample_message)
+ self.assertEqual(self._box.get_file(key0).read().replace(os.linesep, '\n'),
+ self._template % 0)
+ self.assertEqual(self._box.get_file(key1).read().replace(os.linesep, '\n'),
+ _sample_message)
+
+ def test_iterkeys(self):
+ # Get keys using iterkeys()
+ self._check_iteration(self._box.iterkeys, do_keys=True, do_values=False)
+
+ def test_keys(self):
+ # Get keys using keys()
+ self._check_iteration(self._box.keys, do_keys=True, do_values=False)
+
+ def test_itervalues(self):
+ # Get values using itervalues()
+ self._check_iteration(self._box.itervalues, do_keys=False,
+ do_values=True)
+
+ def test_iter(self):
+ # Get values using __iter__()
+ self._check_iteration(self._box.__iter__, do_keys=False,
+ do_values=True)
+
+ def test_values(self):
+ # Get values using values()
+ self._check_iteration(self._box.values, do_keys=False, do_values=True)
+
+ def test_iteritems(self):
+ # Get keys and values using iteritems()
+ self._check_iteration(self._box.iteritems, do_keys=True,
+ do_values=True)
+
+ def test_items(self):
+ # Get keys and values using items()
+ self._check_iteration(self._box.items, do_keys=True, do_values=True)
+
+ def _check_iteration(self, method, do_keys, do_values, repetitions=10):
+ for value in method():
+ self.fail("Not empty")
+ keys, values = [], []
+ for i in xrange(repetitions):
+ keys.append(self._box.add(self._template % i))
+ values.append(self._template % i)
+ if do_keys and not do_values:
+ returned_keys = list(method())
+ elif do_values and not do_keys:
+ returned_values = list(method())
+ else:
+ returned_keys, returned_values = [], []
+ for key, value in method():
+ returned_keys.append(key)
+ returned_values.append(value)
+ if do_keys:
+ self.assertEqual(len(keys), len(returned_keys))
+ self.assertEqual(set(keys), set(returned_keys))
+ if do_values:
+ count = 0
+ for value in returned_values:
+ self.assertEqual(value['from'], 'foo')
+ self.assertTrue(int(value.get_payload()) < repetitions,
+ (value.get_payload(), repetitions))
+ count += 1
+ self.assertEqual(len(values), count)
+
+ def test_has_key(self):
+ # Check existence of keys using has_key()
+ self._test_has_key_or_contains(self._box.has_key)
+
+ def test_contains(self):
+ # Check existence of keys using __contains__()
+ self._test_has_key_or_contains(self._box.__contains__)
+
+ def _test_has_key_or_contains(self, method):
+ # (Used by test_has_key() and test_contains().)
+ self.assertFalse(method('foo'))
+ key0 = self._box.add(self._template % 0)
+ self.assertTrue(method(key0))
+ self.assertFalse(method('foo'))
+ key1 = self._box.add(self._template % 1)
+ self.assertTrue(method(key1))
+ self.assertTrue(method(key0))
+ self.assertFalse(method('foo'))
+ self._box.remove(key0)
+ self.assertFalse(method(key0))
+ self.assertTrue(method(key1))
+ self.assertFalse(method('foo'))
+ self._box.remove(key1)
+ self.assertFalse(method(key1))
+ self.assertFalse(method(key0))
+ self.assertFalse(method('foo'))
+
+ def test_len(self, repetitions=10):
+ # Get message count
+ keys = []
+ for i in xrange(repetitions):
+ self.assertEqual(len(self._box), i)
+ keys.append(self._box.add(self._template % i))
+ self.assertEqual(len(self._box), i + 1)
+ for i in xrange(repetitions):
+ self.assertEqual(len(self._box), repetitions - i)
+ self._box.remove(keys[i])
+ self.assertEqual(len(self._box), repetitions - i - 1)
+
+ def test_set_item(self):
+ # Modify messages using __setitem__()
+ key0 = self._box.add(self._template % 'original 0')
+ self.assertEqual(self._box.get_string(key0),
+ self._template % 'original 0')
+ key1 = self._box.add(self._template % 'original 1')
+ self.assertEqual(self._box.get_string(key1),
+ self._template % 'original 1')
+ self._box[key0] = self._template % 'changed 0'
+ self.assertEqual(self._box.get_string(key0),
+ self._template % 'changed 0')
+ self._box[key1] = self._template % 'changed 1'
+ self.assertEqual(self._box.get_string(key1),
+ self._template % 'changed 1')
+ self._box[key0] = _sample_message
+ self._check_sample(self._box[key0])
+ self._box[key1] = self._box[key0]
+ self._check_sample(self._box[key1])
+ self._box[key0] = self._template % 'original 0'
+ self.assertEqual(self._box.get_string(key0),
+ self._template % 'original 0')
+ self._check_sample(self._box[key1])
+ self.assertRaises(KeyError,
+ lambda: self._box.__setitem__('foo', 'bar'))
+ self.assertRaises(KeyError, lambda: self._box['foo'])
+ self.assertEqual(len(self._box), 2)
+
+ def test_clear(self, iterations=10):
+ # Remove all messages using clear()
+ keys = []
+ for i in xrange(iterations):
+ self._box.add(self._template % i)
+ for i, key in enumerate(keys):
+ self.assertEqual(self._box.get_string(key), self._template % i)
+ self._box.clear()
+ self.assertEqual(len(self._box), 0)
+ for i, key in enumerate(keys):
+ self.assertRaises(KeyError, lambda: self._box.get_string(key))
+
+ def test_pop(self):
+ # Get and remove a message using pop()
+ key0 = self._box.add(self._template % 0)
+ self.assertIn(key0, self._box)
+ key1 = self._box.add(self._template % 1)
+ self.assertIn(key1, self._box)
+ self.assertEqual(self._box.pop(key0).get_payload(), '0')
+ self.assertNotIn(key0, self._box)
+ self.assertIn(key1, self._box)
+ key2 = self._box.add(self._template % 2)
+ self.assertIn(key2, self._box)
+ self.assertEqual(self._box.pop(key2).get_payload(), '2')
+ self.assertNotIn(key2, self._box)
+ self.assertIn(key1, self._box)
+ self.assertEqual(self._box.pop(key1).get_payload(), '1')
+ self.assertNotIn(key1, self._box)
+ self.assertEqual(len(self._box), 0)
+
+ def test_popitem(self, iterations=10):
+ # Get and remove an arbitrary (key, message) using popitem()
+ keys = []
+ for i in xrange(10):
+ keys.append(self._box.add(self._template % i))
+ seen = []
+ for i in xrange(10):
+ key, msg = self._box.popitem()
+ self.assertIn(key, keys)
+ self.assertNotIn(key, seen)
+ seen.append(key)
+ self.assertEqual(int(msg.get_payload()), keys.index(key))
+ self.assertEqual(len(self._box), 0)
+ for key in keys:
+ self.assertRaises(KeyError, lambda: self._box[key])
+
+ def test_update(self):
+ # Modify multiple messages using update()
+ key0 = self._box.add(self._template % 'original 0')
+ key1 = self._box.add(self._template % 'original 1')
+ key2 = self._box.add(self._template % 'original 2')
+ self._box.update({key0: self._template % 'changed 0',
+ key2: _sample_message})
+ self.assertEqual(len(self._box), 3)
+ self.assertEqual(self._box.get_string(key0),
+ self._template % 'changed 0')
+ self.assertEqual(self._box.get_string(key1),
+ self._template % 'original 1')
+ self._check_sample(self._box[key2])
+ self._box.update([(key2, self._template % 'changed 2'),
+ (key1, self._template % 'changed 1'),
+ (key0, self._template % 'original 0')])
+ self.assertEqual(len(self._box), 3)
+ self.assertEqual(self._box.get_string(key0),
+ self._template % 'original 0')
+ self.assertEqual(self._box.get_string(key1),
+ self._template % 'changed 1')
+ self.assertEqual(self._box.get_string(key2),
+ self._template % 'changed 2')
+ self.assertRaises(KeyError,
+ lambda: self._box.update({'foo': 'bar',
+ key0: self._template % "changed 0"}))
+ self.assertEqual(len(self._box), 3)
+ self.assertEqual(self._box.get_string(key0),
+ self._template % "changed 0")
+ self.assertEqual(self._box.get_string(key1),
+ self._template % "changed 1")
+ self.assertEqual(self._box.get_string(key2),
+ self._template % "changed 2")
+
+ def test_flush(self):
+ # Write changes to disk
+ self._test_flush_or_close(self._box.flush, True)
+
+ def test_lock_unlock(self):
+ # Lock and unlock the mailbox
+ self.assertFalse(os.path.exists(self._get_lock_path()))
+ self._box.lock()
+ self.assertTrue(os.path.exists(self._get_lock_path()))
+ self._box.unlock()
+ self.assertFalse(os.path.exists(self._get_lock_path()))
+
+ def test_close(self):
+ # Close mailbox and flush changes to disk
+ self._test_flush_or_close(self._box.close, False)
+
+ def _test_flush_or_close(self, method, should_call_close):
+ contents = [self._template % i for i in xrange(3)]
+ self._box.add(contents[0])
+ self._box.add(contents[1])
+ self._box.add(contents[2])
+ method()
+ if should_call_close:
+ self._box.close()
+ self._box = self._factory(self._path)
+ keys = self._box.keys()
+ self.assertEqual(len(keys), 3)
+ for key in keys:
+ self.assertIn(self._box.get_string(key), contents)
+
+ def test_dump_message(self):
+ # Write message representations to disk
+ for input in (email.message_from_string(_sample_message),
+ _sample_message, StringIO.StringIO(_sample_message)):
+ output = StringIO.StringIO()
+ self._box._dump_message(input, output)
+ self.assertEqual(output.getvalue(),
+ _sample_message.replace('\n', os.linesep))
+ output = StringIO.StringIO()
+ self.assertRaises(TypeError,
+ lambda: self._box._dump_message(None, output))
+
+ def _get_lock_path(self):
+ # Return the path of the dot lock file. May be overridden.
+ return self._path + '.lock'
+
+
+class TestMailboxSuperclass(TestBase):
+
+ def test_notimplemented(self):
+ # Test that all Mailbox methods raise NotImplementedException.
+ box = mailbox.Mailbox('path')
+ self.assertRaises(NotImplementedError, lambda: box.add(''))
+ self.assertRaises(NotImplementedError, lambda: box.remove(''))
+ self.assertRaises(NotImplementedError, lambda: box.__delitem__(''))
+ self.assertRaises(NotImplementedError, lambda: box.discard(''))
+ self.assertRaises(NotImplementedError, lambda: box.__setitem__('', ''))
+ self.assertRaises(NotImplementedError, lambda: box.iterkeys())
+ self.assertRaises(NotImplementedError, lambda: box.keys())
+ self.assertRaises(NotImplementedError, lambda: box.itervalues().next())
+ self.assertRaises(NotImplementedError, lambda: box.__iter__().next())
+ self.assertRaises(NotImplementedError, lambda: box.values())
+ self.assertRaises(NotImplementedError, lambda: box.iteritems().next())
+ self.assertRaises(NotImplementedError, lambda: box.items())
+ self.assertRaises(NotImplementedError, lambda: box.get(''))
+ self.assertRaises(NotImplementedError, lambda: box.__getitem__(''))
+ self.assertRaises(NotImplementedError, lambda: box.get_message(''))
+ self.assertRaises(NotImplementedError, lambda: box.get_string(''))
+ self.assertRaises(NotImplementedError, lambda: box.get_file(''))
+ self.assertRaises(NotImplementedError, lambda: box.has_key(''))
+ self.assertRaises(NotImplementedError, lambda: box.__contains__(''))
+ self.assertRaises(NotImplementedError, lambda: box.__len__())
+ self.assertRaises(NotImplementedError, lambda: box.clear())
+ self.assertRaises(NotImplementedError, lambda: box.pop(''))
+ self.assertRaises(NotImplementedError, lambda: box.popitem())
+ self.assertRaises(NotImplementedError, lambda: box.update((('', ''),)))
+ self.assertRaises(NotImplementedError, lambda: box.flush())
+ self.assertRaises(NotImplementedError, lambda: box.lock())
+ self.assertRaises(NotImplementedError, lambda: box.unlock())
+ self.assertRaises(NotImplementedError, lambda: box.close())
+
+
+class TestMaildir(TestMailbox):
+
+ _factory = lambda self, path, factory=None: mailbox.Maildir(path, factory)
+
+ def setUp(self):
+ TestMailbox.setUp(self)
+ if os.name in ('nt', 'os2') or sys.platform == 'cygwin':
+ self._box.colon = '!'
+
+ def test_add_MM(self):
+ # Add a MaildirMessage instance
+ msg = mailbox.MaildirMessage(self._template % 0)
+ msg.set_subdir('cur')
+ msg.set_info('foo')
+ key = self._box.add(msg)
+ self.assertTrue(os.path.exists(os.path.join(self._path, 'cur', '%s%sfoo' %
+ (key, self._box.colon))))
+
+ def test_get_MM(self):
+ # Get a MaildirMessage instance
+ msg = mailbox.MaildirMessage(self._template % 0)
+ msg.set_subdir('cur')
+ msg.set_flags('RF')
+ key = self._box.add(msg)
+ msg_returned = self._box.get_message(key)
+ self.assertIsInstance(msg_returned, mailbox.MaildirMessage)
+ self.assertEqual(msg_returned.get_subdir(), 'cur')
+ self.assertEqual(msg_returned.get_flags(), 'FR')
+
+ def test_set_MM(self):
+ # Set with a MaildirMessage instance
+ msg0 = mailbox.MaildirMessage(self._template % 0)
+ msg0.set_flags('TP')
+ key = self._box.add(msg0)
+ msg_returned = self._box.get_message(key)
+ self.assertEqual(msg_returned.get_subdir(), 'new')
+ self.assertEqual(msg_returned.get_flags(), 'PT')
+ msg1 = mailbox.MaildirMessage(self._template % 1)
+ self._box[key] = msg1
+ msg_returned = self._box.get_message(key)
+ self.assertEqual(msg_returned.get_subdir(), 'new')
+ self.assertEqual(msg_returned.get_flags(), '')
+ self.assertEqual(msg_returned.get_payload(), '1')
+ msg2 = mailbox.MaildirMessage(self._template % 2)
+ msg2.set_info('2,S')
+ self._box[key] = msg2
+ self._box[key] = self._template % 3
+ msg_returned = self._box.get_message(key)
+ self.assertEqual(msg_returned.get_subdir(), 'new')
+ self.assertEqual(msg_returned.get_flags(), 'S')
+ self.assertEqual(msg_returned.get_payload(), '3')
+
+ def test_consistent_factory(self):
+ # Add a message.
+ msg = mailbox.MaildirMessage(self._template % 0)
+ msg.set_subdir('cur')
+ msg.set_flags('RF')
+ key = self._box.add(msg)
+
+ # Create new mailbox with
+ class FakeMessage(mailbox.MaildirMessage):
+ pass
+ box = mailbox.Maildir(self._path, factory=FakeMessage)
+ box.colon = self._box.colon
+ msg2 = box.get_message(key)
+ self.assertIsInstance(msg2, FakeMessage)
+
+ def test_initialize_new(self):
+ # Initialize a non-existent mailbox
+ self.tearDown()
+ self._box = mailbox.Maildir(self._path)
+ self._check_basics(factory=rfc822.Message)
+ self._delete_recursively(self._path)
+ self._box = self._factory(self._path, factory=None)
+ self._check_basics()
+
+ def test_initialize_existing(self):
+ # Initialize an existing mailbox
+ self.tearDown()
+ for subdir in '', 'tmp', 'new', 'cur':
+ os.mkdir(os.path.normpath(os.path.join(self._path, subdir)))
+ self._box = mailbox.Maildir(self._path)
+ self._check_basics(factory=rfc822.Message)
+ self._box = mailbox.Maildir(self._path, factory=None)
+ self._check_basics()
+
+ def _check_basics(self, factory=None):
+ # (Used by test_open_new() and test_open_existing().)
+ self.assertEqual(self._box._path, os.path.abspath(self._path))
+ self.assertEqual(self._box._factory, factory)
+ for subdir in '', 'tmp', 'new', 'cur':
+ path = os.path.join(self._path, subdir)
+ mode = os.stat(path)[stat.ST_MODE]
+ self.assertTrue(stat.S_ISDIR(mode), "Not a directory: '%s'" % path)
+
+ def test_list_folders(self):
+ # List folders
+ self._box.add_folder('one')
+ self._box.add_folder('two')
+ self._box.add_folder('three')
+ self.assertEqual(len(self._box.list_folders()), 3)
+ self.assertEqual(set(self._box.list_folders()),
+ set(('one', 'two', 'three')))
+
+ def test_get_folder(self):
+ # Open folders
+ self._box.add_folder('foo.bar')
+ folder0 = self._box.get_folder('foo.bar')
+ folder0.add(self._template % 'bar')
+ self.assertTrue(os.path.isdir(os.path.join(self._path, '.foo.bar')))
+ folder1 = self._box.get_folder('foo.bar')
+ self.assertEqual(folder1.get_string(folder1.keys()[0]),
+ self._template % 'bar')
+
+ def test_add_and_remove_folders(self):
+ # Delete folders
+ self._box.add_folder('one')
+ self._box.add_folder('two')
+ self.assertEqual(len(self._box.list_folders()), 2)
+ self.assertEqual(set(self._box.list_folders()), set(('one', 'two')))
+ self._box.remove_folder('one')
+ self.assertEqual(len(self._box.list_folders()), 1)
+ self.assertEqual(set(self._box.list_folders()), set(('two',)))
+ self._box.add_folder('three')
+ self.assertEqual(len(self._box.list_folders()), 2)
+ self.assertEqual(set(self._box.list_folders()), set(('two', 'three')))
+ self._box.remove_folder('three')
+ self.assertEqual(len(self._box.list_folders()), 1)
+ self.assertEqual(set(self._box.list_folders()), set(('two',)))
+ self._box.remove_folder('two')
+ self.assertEqual(len(self._box.list_folders()), 0)
+ self.assertEqual(self._box.list_folders(), [])
+
+ def test_clean(self):
+ # Remove old files from 'tmp'
+ foo_path = os.path.join(self._path, 'tmp', 'foo')
+ bar_path = os.path.join(self._path, 'tmp', 'bar')
+ with open(foo_path, 'w') as f:
+ f.write("@")
+ with open(bar_path, 'w') as f:
+ f.write("@")
+ self._box.clean()
+ self.assertTrue(os.path.exists(foo_path))
+ self.assertTrue(os.path.exists(bar_path))
+ foo_stat = os.stat(foo_path)
+ os.utime(foo_path, (time.time() - 129600 - 2,
+ foo_stat.st_mtime))
+ self._box.clean()
+ self.assertFalse(os.path.exists(foo_path))
+ self.assertTrue(os.path.exists(bar_path))
+
+ def test_create_tmp(self, repetitions=10):
+ # Create files in tmp directory
+ hostname = socket.gethostname()
+ if '/' in hostname:
+ hostname = hostname.replace('/', r'\057')
+ if ':' in hostname:
+ hostname = hostname.replace(':', r'\072')
+ pid = os.getpid()
+ pattern = re.compile(r"(?P<time>\d+)\.M(?P<M>\d{1,6})P(?P<P>\d+)"
+ r"Q(?P<Q>\d+)\.(?P<host>[^:/]+)")
+ previous_groups = None
+ for x in xrange(repetitions):
+ tmp_file = self._box._create_tmp()
+ head, tail = os.path.split(tmp_file.name)
+ self.assertEqual(head, os.path.abspath(os.path.join(self._path,
+ "tmp")),
+ "File in wrong location: '%s'" % head)
+ match = pattern.match(tail)
+ self.assertTrue(match is not None, "Invalid file name: '%s'" % tail)
+ groups = match.groups()
+ if previous_groups is not None:
+ self.assertTrue(int(groups[0] >= previous_groups[0]),
+ "Non-monotonic seconds: '%s' before '%s'" %
+ (previous_groups[0], groups[0]))
+ self.assertTrue(int(groups[1] >= previous_groups[1]) or
+ groups[0] != groups[1],
+ "Non-monotonic milliseconds: '%s' before '%s'" %
+ (previous_groups[1], groups[1]))
+ self.assertTrue(int(groups[2]) == pid,
+ "Process ID mismatch: '%s' should be '%s'" %
+ (groups[2], pid))
+ self.assertTrue(int(groups[3]) == int(previous_groups[3]) + 1,
+ "Non-sequential counter: '%s' before '%s'" %
+ (previous_groups[3], groups[3]))
+ self.assertTrue(groups[4] == hostname,
+ "Host name mismatch: '%s' should be '%s'" %
+ (groups[4], hostname))
+ previous_groups = groups
+ tmp_file.write(_sample_message)
+ tmp_file.seek(0)
+ self.assertTrue(tmp_file.read() == _sample_message)
+ tmp_file.close()
+ file_count = len(os.listdir(os.path.join(self._path, "tmp")))
+ self.assertTrue(file_count == repetitions,
+ "Wrong file count: '%s' should be '%s'" %
+ (file_count, repetitions))
+
+ def test_refresh(self):
+ # Update the table of contents
+ self.assertEqual(self._box._toc, {})
+ key0 = self._box.add(self._template % 0)
+ key1 = self._box.add(self._template % 1)
+ self.assertEqual(self._box._toc, {})
+ self._box._refresh()
+ self.assertEqual(self._box._toc, {key0: os.path.join('new', key0),
+ key1: os.path.join('new', key1)})
+ key2 = self._box.add(self._template % 2)
+ self.assertEqual(self._box._toc, {key0: os.path.join('new', key0),
+ key1: os.path.join('new', key1)})
+ self._box._refresh()
+ self.assertEqual(self._box._toc, {key0: os.path.join('new', key0),
+ key1: os.path.join('new', key1),
+ key2: os.path.join('new', key2)})
+
+ def test_lookup(self):
+ # Look up message subpaths in the TOC
+ self.assertRaises(KeyError, lambda: self._box._lookup('foo'))
+ key0 = self._box.add(self._template % 0)
+ self.assertEqual(self._box._lookup(key0), os.path.join('new', key0))
+ os.remove(os.path.join(self._path, 'new', key0))
+ self.assertEqual(self._box._toc, {key0: os.path.join('new', key0)})
+ # Be sure that the TOC is read back from disk (see issue #6896
+ # about bad mtime behaviour on some systems).
+ self._box.flush()
+ self.assertRaises(KeyError, lambda: self._box._lookup(key0))
+ self.assertEqual(self._box._toc, {})
+
+ def test_lock_unlock(self):
+ # Lock and unlock the mailbox. For Maildir, this does nothing.
+ self._box.lock()
+ self._box.unlock()
+
+ def test_folder (self):
+ # Test for bug #1569790: verify that folders returned by .get_folder()
+ # use the same factory function.
+ def dummy_factory (s):
+ return None
+ box = self._factory(self._path, factory=dummy_factory)
+ folder = box.add_folder('folder1')
+ self.assertIs(folder._factory, dummy_factory)
+
+ folder1_alias = box.get_folder('folder1')
+ self.assertIs(folder1_alias._factory, dummy_factory)
+
+ def test_directory_in_folder (self):
+ # Test that mailboxes still work if there's a stray extra directory
+ # in a folder.
+ for i in range(10):
+ self._box.add(mailbox.Message(_sample_message))
+
+ # Create a stray directory
+ os.mkdir(os.path.join(self._path, 'cur', 'stray-dir'))
+
+ # Check that looping still works with the directory present.
+ for msg in self._box:
+ pass
+
+ def test_file_permissions(self):
+ # Verify that message files are created without execute permissions
+ if not hasattr(os, "stat") or not hasattr(os, "umask"):
+ return
+ msg = mailbox.MaildirMessage(self._template % 0)
+ orig_umask = os.umask(0)
+ try:
+ key = self._box.add(msg)
+ finally:
+ os.umask(orig_umask)
+ path = os.path.join(self._path, self._box._lookup(key))
+ mode = os.stat(path).st_mode
+ self.assertEqual(mode & 0111, 0)
+
+ def test_folder_file_perms(self):
+ # From bug #3228, we want to verify that the file created inside a Maildir
+ # subfolder isn't marked as executable.
+ if not hasattr(os, "stat") or not hasattr(os, "umask"):
+ return
+
+ orig_umask = os.umask(0)
+ try:
+ subfolder = self._box.add_folder('subfolder')
+ finally:
+ os.umask(orig_umask)
+
+ path = os.path.join(subfolder._path, 'maildirfolder')
+ st = os.stat(path)
+ perms = st.st_mode
+ self.assertFalse((perms & 0111)) # Execute bits should all be off.
+
+ def test_reread(self):
+
+ # Put the last modified times more than two seconds into the past
+ # (because mtime may have only a two second granularity).
+ for subdir in ('cur', 'new'):
+ os.utime(os.path.join(self._box._path, subdir),
+ (time.time()-5,)*2)
+
+ # Because mtime has a two second granularity in worst case (FAT), a
+ # refresh is done unconditionally if called for within
+ # two-second-plus-a-bit of the last one, just in case the mbox has
+ # changed; so now we have to wait for that interval to expire.
+ time.sleep(2.01 + self._box._skewfactor)
+
+ # Re-reading causes the ._toc attribute to be assigned a new dictionary
+ # object, so we'll check that the ._toc attribute isn't a different
+ # object.
+ orig_toc = self._box._toc
+ def refreshed():
+ return self._box._toc is not orig_toc
+
+ self._box._refresh()
+ self.assertFalse(refreshed())
+
+ # Now, write something into cur and remove it. This changes
+ # the mtime and should cause a re-read.
+ filename = os.path.join(self._path, 'cur', 'stray-file')
+ f = open(filename, 'w')
+ f.close()
+ os.unlink(filename)
+ self._box._refresh()
+ self.assertTrue(refreshed())
+
+class _TestMboxMMDF(TestMailbox):
+
+ def tearDown(self):
+ self._box.close()
+ self._delete_recursively(self._path)
+ for lock_remnant in glob.glob(self._path + '.*'):
+ test_support.unlink(lock_remnant)
+
+ def test_add_from_string(self):
+ # Add a string starting with 'From ' to the mailbox
+ key = self._box.add('From foo at bar blah\nFrom: foo\n\n0')
+ self.assertEqual(self._box[key].get_from(), 'foo at bar blah')
+ self.assertEqual(self._box[key].get_payload(), '0')
+
+ def test_add_mbox_or_mmdf_message(self):
+ # Add an mboxMessage or MMDFMessage
+ for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+ msg = class_('From foo at bar blah\nFrom: foo\n\n0')
+ key = self._box.add(msg)
+
+ def test_open_close_open(self):
+ # Open and inspect previously-created mailbox
+ values = [self._template % i for i in xrange(3)]
+ for value in values:
+ self._box.add(value)
+ self._box.close()
+ mtime = os.path.getmtime(self._path)
+ self._box = self._factory(self._path)
+ self.assertEqual(len(self._box), 3)
+ for key in self._box.iterkeys():
+ self.assertIn(self._box.get_string(key), values)
+ self._box.close()
+ self.assertEqual(mtime, os.path.getmtime(self._path))
+
+ def test_add_and_close(self):
+ # Verifying that closing a mailbox doesn't change added items
+ self._box.add(_sample_message)
+ for i in xrange(3):
+ self._box.add(self._template % i)
+ self._box.add(_sample_message)
+ self._box._file.flush()
+ self._box._file.seek(0)
+ contents = self._box._file.read()
+ self._box.close()
+ with open(self._path, 'rb') as f:
+ self.assertEqual(contents, f.read())
+ self._box = self._factory(self._path)
+
+ def test_lock_conflict(self):
+ # Fork off a subprocess that will lock the file for 2 seconds,
+ # unlock it, and then exit.
+ if not hasattr(os, 'fork'):
+ return
+ pid = os.fork()
+ if pid == 0:
+ # In the child, lock the mailbox.
+ self._box.lock()
+ time.sleep(2)
+ self._box.unlock()
+ os._exit(0)
+
+ # In the parent, sleep a bit to give the child time to acquire
+ # the lock.
+ time.sleep(0.5)
+ try:
+ self.assertRaises(mailbox.ExternalClashError,
+ self._box.lock)
+ finally:
+ # Wait for child to exit. Locking should now succeed.
+ exited_pid, status = os.waitpid(pid, 0)
+
+ self._box.lock()
+ self._box.unlock()
+
+ def test_relock(self):
+ # Test case for bug #1575506: the mailbox class was locking the
+ # wrong file object in its flush() method.
+ msg = "Subject: sub\n\nbody\n"
+ key1 = self._box.add(msg)
+ self._box.flush()
+ self._box.close()
+
+ self._box = self._factory(self._path)
+ self._box.lock()
+ key2 = self._box.add(msg)
+ self._box.flush()
+ self.assertTrue(self._box._locked)
+ self._box.close()
+
+
+class TestMbox(_TestMboxMMDF):
+
+ _factory = lambda self, path, factory=None: mailbox.mbox(path, factory)
+
+ def test_file_perms(self):
+ # From bug #3228, we want to verify that the mailbox file isn't executable,
+ # even if the umask is set to something that would leave executable bits set.
+ # We only run this test on platforms that support umask.
+ if hasattr(os, 'umask') and hasattr(os, 'stat'):
+ try:
+ old_umask = os.umask(0077)
+ self._box.close()
+ os.unlink(self._path)
+ self._box = mailbox.mbox(self._path, create=True)
+ self._box.add('')
+ self._box.close()
+ finally:
+ os.umask(old_umask)
+
+ st = os.stat(self._path)
+ perms = st.st_mode
+ self.assertFalse((perms & 0111)) # Execute bits should all be off.
+
+class TestMMDF(_TestMboxMMDF):
+
+ _factory = lambda self, path, factory=None: mailbox.MMDF(path, factory)
+
+
+class TestMH(TestMailbox):
+
+ _factory = lambda self, path, factory=None: mailbox.MH(path, factory)
+
+ def test_list_folders(self):
+ # List folders
+ self._box.add_folder('one')
+ self._box.add_folder('two')
+ self._box.add_folder('three')
+ self.assertEqual(len(self._box.list_folders()), 3)
+ self.assertEqual(set(self._box.list_folders()),
+ set(('one', 'two', 'three')))
+
+ def test_get_folder(self):
+ # Open folders
+ def dummy_factory (s):
+ return None
+ self._box = self._factory(self._path, dummy_factory)
+
+ new_folder = self._box.add_folder('foo.bar')
+ folder0 = self._box.get_folder('foo.bar')
+ folder0.add(self._template % 'bar')
+ self.assertTrue(os.path.isdir(os.path.join(self._path, 'foo.bar')))
+ folder1 = self._box.get_folder('foo.bar')
+ self.assertEqual(folder1.get_string(folder1.keys()[0]),
+ self._template % 'bar')
+
+ # Test for bug #1569790: verify that folders returned by .get_folder()
+ # use the same factory function.
+ self.assertIs(new_folder._factory, self._box._factory)
+ self.assertIs(folder0._factory, self._box._factory)
+
+ def test_add_and_remove_folders(self):
+ # Delete folders
+ self._box.add_folder('one')
+ self._box.add_folder('two')
+ self.assertEqual(len(self._box.list_folders()), 2)
+ self.assertEqual(set(self._box.list_folders()), set(('one', 'two')))
+ self._box.remove_folder('one')
+ self.assertEqual(len(self._box.list_folders()), 1)
+ self.assertEqual(set(self._box.list_folders()), set(('two', )))
+ self._box.add_folder('three')
+ self.assertEqual(len(self._box.list_folders()), 2)
+ self.assertEqual(set(self._box.list_folders()), set(('two', 'three')))
+ self._box.remove_folder('three')
+ self.assertEqual(len(self._box.list_folders()), 1)
+ self.assertEqual(set(self._box.list_folders()), set(('two', )))
+ self._box.remove_folder('two')
+ self.assertEqual(len(self._box.list_folders()), 0)
+ self.assertEqual(self._box.list_folders(), [])
+
+ def test_sequences(self):
+ # Get and set sequences
+ self.assertEqual(self._box.get_sequences(), {})
+ msg0 = mailbox.MHMessage(self._template % 0)
+ msg0.add_sequence('foo')
+ key0 = self._box.add(msg0)
+ self.assertEqual(self._box.get_sequences(), {'foo':[key0]})
+ msg1 = mailbox.MHMessage(self._template % 1)
+ msg1.set_sequences(['bar', 'replied', 'foo'])
+ key1 = self._box.add(msg1)
+ self.assertEqual(self._box.get_sequences(),
+ {'foo':[key0, key1], 'bar':[key1], 'replied':[key1]})
+ msg0.set_sequences(['flagged'])
+ self._box[key0] = msg0
+ self.assertEqual(self._box.get_sequences(),
+ {'foo':[key1], 'bar':[key1], 'replied':[key1],
+ 'flagged':[key0]})
+ self._box.remove(key1)
+ self.assertEqual(self._box.get_sequences(), {'flagged':[key0]})
+
+ def test_issue2625(self):
+ msg0 = mailbox.MHMessage(self._template % 0)
+ msg0.add_sequence('foo')
+ key0 = self._box.add(msg0)
+ refmsg0 = self._box.get_message(key0)
+
+ def test_issue7627(self):
+ msg0 = mailbox.MHMessage(self._template % 0)
+ key0 = self._box.add(msg0)
+ self._box.lock()
+ self._box.remove(key0)
+ self._box.unlock()
+
+ def test_pack(self):
+ # Pack the contents of the mailbox
+ msg0 = mailbox.MHMessage(self._template % 0)
+ msg1 = mailbox.MHMessage(self._template % 1)
+ msg2 = mailbox.MHMessage(self._template % 2)
+ msg3 = mailbox.MHMessage(self._template % 3)
+ msg0.set_sequences(['foo', 'unseen'])
+ msg1.set_sequences(['foo'])
+ msg2.set_sequences(['foo', 'flagged'])
+ msg3.set_sequences(['foo', 'bar', 'replied'])
+ key0 = self._box.add(msg0)
+ key1 = self._box.add(msg1)
+ key2 = self._box.add(msg2)
+ key3 = self._box.add(msg3)
+ self.assertEqual(self._box.get_sequences(),
+ {'foo':[key0,key1,key2,key3], 'unseen':[key0],
+ 'flagged':[key2], 'bar':[key3], 'replied':[key3]})
+ self._box.remove(key2)
+ self.assertEqual(self._box.get_sequences(),
+ {'foo':[key0,key1,key3], 'unseen':[key0], 'bar':[key3],
+ 'replied':[key3]})
+ self._box.pack()
+ self.assertEqual(self._box.keys(), [1, 2, 3])
+ key0 = key0
+ key1 = key0 + 1
+ key2 = key1 + 1
+ self.assertEqual(self._box.get_sequences(),
+ {'foo':[1, 2, 3], 'unseen':[1], 'bar':[3], 'replied':[3]})
+
+ # Test case for packing while holding the mailbox locked.
+ key0 = self._box.add(msg1)
+ key1 = self._box.add(msg1)
+ key2 = self._box.add(msg1)
+ key3 = self._box.add(msg1)
+
+ self._box.remove(key0)
+ self._box.remove(key2)
+ self._box.lock()
+ self._box.pack()
+ self._box.unlock()
+ self.assertEqual(self._box.get_sequences(),
+ {'foo':[1, 2, 3, 4, 5],
+ 'unseen':[1], 'bar':[3], 'replied':[3]})
+
+ def _get_lock_path(self):
+ return os.path.join(self._path, '.mh_sequences.lock')
+
+
+class TestBabyl(TestMailbox):
+
+ _factory = lambda self, path, factory=None: mailbox.Babyl(path, factory)
+
+ def tearDown(self):
+ self._box.close()
+ self._delete_recursively(self._path)
+ for lock_remnant in glob.glob(self._path + '.*'):
+ test_support.unlink(lock_remnant)
+
+ def test_labels(self):
+ # Get labels from the mailbox
+ self.assertEqual(self._box.get_labels(), [])
+ msg0 = mailbox.BabylMessage(self._template % 0)
+ msg0.add_label('foo')
+ key0 = self._box.add(msg0)
+ self.assertEqual(self._box.get_labels(), ['foo'])
+ msg1 = mailbox.BabylMessage(self._template % 1)
+ msg1.set_labels(['bar', 'answered', 'foo'])
+ key1 = self._box.add(msg1)
+ self.assertEqual(set(self._box.get_labels()), set(['foo', 'bar']))
+ msg0.set_labels(['blah', 'filed'])
+ self._box[key0] = msg0
+ self.assertEqual(set(self._box.get_labels()),
+ set(['foo', 'bar', 'blah']))
+ self._box.remove(key1)
+ self.assertEqual(set(self._box.get_labels()), set(['blah']))
+
+
+class TestMessage(TestBase):
+
+ _factory = mailbox.Message # Overridden by subclasses to reuse tests
+
+ def setUp(self):
+ self._path = test_support.TESTFN
+
+ def tearDown(self):
+ self._delete_recursively(self._path)
+
+ def test_initialize_with_eMM(self):
+ # Initialize based on email.message.Message instance
+ eMM = email.message_from_string(_sample_message)
+ msg = self._factory(eMM)
+ self._post_initialize_hook(msg)
+ self._check_sample(msg)
+
+ def test_initialize_with_string(self):
+ # Initialize based on string
+ msg = self._factory(_sample_message)
+ self._post_initialize_hook(msg)
+ self._check_sample(msg)
+
+ def test_initialize_with_file(self):
+ # Initialize based on contents of file
+ with open(self._path, 'w+') as f:
+ f.write(_sample_message)
+ f.seek(0)
+ msg = self._factory(f)
+ self._post_initialize_hook(msg)
+ self._check_sample(msg)
+
+ def test_initialize_with_nothing(self):
+ # Initialize without arguments
+ msg = self._factory()
+ self._post_initialize_hook(msg)
+ self.assertIsInstance(msg, email.message.Message)
+ self.assertIsInstance(msg, mailbox.Message)
+ self.assertIsInstance(msg, self._factory)
+ self.assertEqual(msg.keys(), [])
+ self.assertFalse(msg.is_multipart())
+ self.assertEqual(msg.get_payload(), None)
+
+ def test_initialize_incorrectly(self):
+ # Initialize with invalid argument
+ self.assertRaises(TypeError, lambda: self._factory(object()))
+
+ def test_become_message(self):
+ # Take on the state of another message
+ eMM = email.message_from_string(_sample_message)
+ msg = self._factory()
+ msg._become_message(eMM)
+ self._check_sample(msg)
+
+ def test_explain_to(self):
+ # Copy self's format-specific data to other message formats.
+ # This test is superficial; better ones are in TestMessageConversion.
+ msg = self._factory()
+ for class_ in (mailbox.Message, mailbox.MaildirMessage,
+ mailbox.mboxMessage, mailbox.MHMessage,
+ mailbox.BabylMessage, mailbox.MMDFMessage):
+ other_msg = class_()
+ msg._explain_to(other_msg)
+ other_msg = email.message.Message()
+ self.assertRaises(TypeError, lambda: msg._explain_to(other_msg))
+
+ def _post_initialize_hook(self, msg):
+ # Overridden by subclasses to check extra things after initialization
+ pass
+
+
+class TestMaildirMessage(TestMessage):
+
+ _factory = mailbox.MaildirMessage
+
+ def _post_initialize_hook(self, msg):
+ self.assertEqual(msg._subdir, 'new')
+ self.assertEqual(msg._info,'')
+
+ def test_subdir(self):
+ # Use get_subdir() and set_subdir()
+ msg = mailbox.MaildirMessage(_sample_message)
+ self.assertEqual(msg.get_subdir(), 'new')
+ msg.set_subdir('cur')
+ self.assertEqual(msg.get_subdir(), 'cur')
+ msg.set_subdir('new')
+ self.assertEqual(msg.get_subdir(), 'new')
+ self.assertRaises(ValueError, lambda: msg.set_subdir('tmp'))
+ self.assertEqual(msg.get_subdir(), 'new')
+ msg.set_subdir('new')
+ self.assertEqual(msg.get_subdir(), 'new')
+ self._check_sample(msg)
+
+ def test_flags(self):
+ # Use get_flags(), set_flags(), add_flag(), remove_flag()
+ msg = mailbox.MaildirMessage(_sample_message)
+ self.assertEqual(msg.get_flags(), '')
+ self.assertEqual(msg.get_subdir(), 'new')
+ msg.set_flags('F')
+ self.assertEqual(msg.get_subdir(), 'new')
+ self.assertEqual(msg.get_flags(), 'F')
+ msg.set_flags('SDTP')
+ self.assertEqual(msg.get_flags(), 'DPST')
+ msg.add_flag('FT')
+ self.assertEqual(msg.get_flags(), 'DFPST')
+ msg.remove_flag('TDRP')
+ self.assertEqual(msg.get_flags(), 'FS')
+ self.assertEqual(msg.get_subdir(), 'new')
+ self._check_sample(msg)
+
+ def test_date(self):
+ # Use get_date() and set_date()
+ msg = mailbox.MaildirMessage(_sample_message)
+ diff = msg.get_date() - time.time()
+ self.assertTrue(abs(diff) < 60, diff)
+ msg.set_date(0.0)
+ self.assertEqual(msg.get_date(), 0.0)
+
+ def test_info(self):
+ # Use get_info() and set_info()
+ msg = mailbox.MaildirMessage(_sample_message)
+ self.assertEqual(msg.get_info(), '')
+ msg.set_info('1,foo=bar')
+ self.assertEqual(msg.get_info(), '1,foo=bar')
+ self.assertRaises(TypeError, lambda: msg.set_info(None))
+ self._check_sample(msg)
+
+ def test_info_and_flags(self):
+ # Test interaction of info and flag methods
+ msg = mailbox.MaildirMessage(_sample_message)
+ self.assertEqual(msg.get_info(), '')
+ msg.set_flags('SF')
+ self.assertEqual(msg.get_flags(), 'FS')
+ self.assertEqual(msg.get_info(), '2,FS')
+ msg.set_info('1,')
+ self.assertEqual(msg.get_flags(), '')
+ self.assertEqual(msg.get_info(), '1,')
+ msg.remove_flag('RPT')
+ self.assertEqual(msg.get_flags(), '')
+ self.assertEqual(msg.get_info(), '1,')
+ msg.add_flag('D')
+ self.assertEqual(msg.get_flags(), 'D')
+ self.assertEqual(msg.get_info(), '2,D')
+ self._check_sample(msg)
+
+
+class _TestMboxMMDFMessage(TestMessage):
+
+ _factory = mailbox._mboxMMDFMessage
+
+ def _post_initialize_hook(self, msg):
+ self._check_from(msg)
+
+ def test_initialize_with_unixfrom(self):
+ # Initialize with a message that already has a _unixfrom attribute
+ msg = mailbox.Message(_sample_message)
+ msg.set_unixfrom('From foo at bar blah')
+ msg = mailbox.mboxMessage(msg)
+ self.assertEqual(msg.get_from(), 'foo at bar blah')
+
+ def test_from(self):
+ # Get and set "From " line
+ msg = mailbox.mboxMessage(_sample_message)
+ self._check_from(msg)
+ msg.set_from('foo bar')
+ self.assertEqual(msg.get_from(), 'foo bar')
+ msg.set_from('foo at bar', True)
+ self._check_from(msg, 'foo at bar')
+ msg.set_from('blah at temp', time.localtime())
+ self._check_from(msg, 'blah at temp')
+
+ def test_flags(self):
+ # Use get_flags(), set_flags(), add_flag(), remove_flag()
+ msg = mailbox.mboxMessage(_sample_message)
+ self.assertEqual(msg.get_flags(), '')
+ msg.set_flags('F')
+ self.assertEqual(msg.get_flags(), 'F')
+ msg.set_flags('XODR')
+ self.assertEqual(msg.get_flags(), 'RODX')
+ msg.add_flag('FA')
+ self.assertEqual(msg.get_flags(), 'RODFAX')
+ msg.remove_flag('FDXA')
+ self.assertEqual(msg.get_flags(), 'RO')
+ self._check_sample(msg)
+
+ def _check_from(self, msg, sender=None):
+ # Check contents of "From " line
+ if sender is None:
+ sender = "MAILER-DAEMON"
+ self.assertTrue(re.match(sender + r" \w{3} \w{3} [\d ]\d [\d ]\d:\d{2}:"
+ r"\d{2} \d{4}", msg.get_from()))
+
+
+class TestMboxMessage(_TestMboxMMDFMessage):
+
+ _factory = mailbox.mboxMessage
+
+
+class TestMHMessage(TestMessage):
+
+ _factory = mailbox.MHMessage
+
+ def _post_initialize_hook(self, msg):
+ self.assertEqual(msg._sequences, [])
+
+ def test_sequences(self):
+ # Get, set, join, and leave sequences
+ msg = mailbox.MHMessage(_sample_message)
+ self.assertEqual(msg.get_sequences(), [])
+ msg.set_sequences(['foobar'])
+ self.assertEqual(msg.get_sequences(), ['foobar'])
+ msg.set_sequences([])
+ self.assertEqual(msg.get_sequences(), [])
+ msg.add_sequence('unseen')
+ self.assertEqual(msg.get_sequences(), ['unseen'])
+ msg.add_sequence('flagged')
+ self.assertEqual(msg.get_sequences(), ['unseen', 'flagged'])
+ msg.add_sequence('flagged')
+ self.assertEqual(msg.get_sequences(), ['unseen', 'flagged'])
+ msg.remove_sequence('unseen')
+ self.assertEqual(msg.get_sequences(), ['flagged'])
+ msg.add_sequence('foobar')
+ self.assertEqual(msg.get_sequences(), ['flagged', 'foobar'])
+ msg.remove_sequence('replied')
+ self.assertEqual(msg.get_sequences(), ['flagged', 'foobar'])
+ msg.set_sequences(['foobar', 'replied'])
+ self.assertEqual(msg.get_sequences(), ['foobar', 'replied'])
+
+
+class TestBabylMessage(TestMessage):
+
+ _factory = mailbox.BabylMessage
+
+ def _post_initialize_hook(self, msg):
+ self.assertEqual(msg._labels, [])
+
+ def test_labels(self):
+ # Get, set, join, and leave labels
+ msg = mailbox.BabylMessage(_sample_message)
+ self.assertEqual(msg.get_labels(), [])
+ msg.set_labels(['foobar'])
+ self.assertEqual(msg.get_labels(), ['foobar'])
+ msg.set_labels([])
+ self.assertEqual(msg.get_labels(), [])
+ msg.add_label('filed')
+ self.assertEqual(msg.get_labels(), ['filed'])
+ msg.add_label('resent')
+ self.assertEqual(msg.get_labels(), ['filed', 'resent'])
+ msg.add_label('resent')
+ self.assertEqual(msg.get_labels(), ['filed', 'resent'])
+ msg.remove_label('filed')
+ self.assertEqual(msg.get_labels(), ['resent'])
+ msg.add_label('foobar')
+ self.assertEqual(msg.get_labels(), ['resent', 'foobar'])
+ msg.remove_label('unseen')
+ self.assertEqual(msg.get_labels(), ['resent', 'foobar'])
+ msg.set_labels(['foobar', 'answered'])
+ self.assertEqual(msg.get_labels(), ['foobar', 'answered'])
+
+ def test_visible(self):
+ # Get, set, and update visible headers
+ msg = mailbox.BabylMessage(_sample_message)
+ visible = msg.get_visible()
+ self.assertEqual(visible.keys(), [])
+ self.assertIs(visible.get_payload(), None)
+ visible['User-Agent'] = 'FooBar 1.0'
+ visible['X-Whatever'] = 'Blah'
+ self.assertEqual(msg.get_visible().keys(), [])
+ msg.set_visible(visible)
+ visible = msg.get_visible()
+ self.assertEqual(visible.keys(), ['User-Agent', 'X-Whatever'])
+ self.assertEqual(visible['User-Agent'], 'FooBar 1.0')
+ self.assertEqual(visible['X-Whatever'], 'Blah')
+ self.assertIs(visible.get_payload(), None)
+ msg.update_visible()
+ self.assertEqual(visible.keys(), ['User-Agent', 'X-Whatever'])
+ self.assertIs(visible.get_payload(), None)
+ visible = msg.get_visible()
+ self.assertEqual(visible.keys(), ['User-Agent', 'Date', 'From', 'To',
+ 'Subject'])
+ for header in ('User-Agent', 'Date', 'From', 'To', 'Subject'):
+ self.assertEqual(visible[header], msg[header])
+
+
+class TestMMDFMessage(_TestMboxMMDFMessage):
+
+ _factory = mailbox.MMDFMessage
+
+
+class TestMessageConversion(TestBase):
+
+ def test_plain_to_x(self):
+ # Convert Message to all formats
+ for class_ in (mailbox.Message, mailbox.MaildirMessage,
+ mailbox.mboxMessage, mailbox.MHMessage,
+ mailbox.BabylMessage, mailbox.MMDFMessage):
+ msg_plain = mailbox.Message(_sample_message)
+ msg = class_(msg_plain)
+ self._check_sample(msg)
+
+ def test_x_to_plain(self):
+ # Convert all formats to Message
+ for class_ in (mailbox.Message, mailbox.MaildirMessage,
+ mailbox.mboxMessage, mailbox.MHMessage,
+ mailbox.BabylMessage, mailbox.MMDFMessage):
+ msg = class_(_sample_message)
+ msg_plain = mailbox.Message(msg)
+ self._check_sample(msg_plain)
+
+ def test_x_to_invalid(self):
+ # Convert all formats to an invalid format
+ for class_ in (mailbox.Message, mailbox.MaildirMessage,
+ mailbox.mboxMessage, mailbox.MHMessage,
+ mailbox.BabylMessage, mailbox.MMDFMessage):
+ self.assertRaises(TypeError, lambda: class_(False))
+
+ def test_maildir_to_maildir(self):
+ # Convert MaildirMessage to MaildirMessage
+ msg_maildir = mailbox.MaildirMessage(_sample_message)
+ msg_maildir.set_flags('DFPRST')
+ msg_maildir.set_subdir('cur')
+ date = msg_maildir.get_date()
+ msg = mailbox.MaildirMessage(msg_maildir)
+ self._check_sample(msg)
+ self.assertEqual(msg.get_flags(), 'DFPRST')
+ self.assertEqual(msg.get_subdir(), 'cur')
+ self.assertEqual(msg.get_date(), date)
+
+ def test_maildir_to_mboxmmdf(self):
+ # Convert MaildirMessage to mboxmessage and MMDFMessage
+ pairs = (('D', ''), ('F', 'F'), ('P', ''), ('R', 'A'), ('S', 'R'),
+ ('T', 'D'), ('DFPRST', 'RDFA'))
+ for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+ msg_maildir = mailbox.MaildirMessage(_sample_message)
+ msg_maildir.set_date(0.0)
+ for setting, result in pairs:
+ msg_maildir.set_flags(setting)
+ msg = class_(msg_maildir)
+ self.assertEqual(msg.get_flags(), result)
+ self.assertEqual(msg.get_from(), 'MAILER-DAEMON %s' %
+ time.asctime(time.gmtime(0.0)))
+ msg_maildir.set_subdir('cur')
+ self.assertEqual(class_(msg_maildir).get_flags(), 'RODFA')
+
+ def test_maildir_to_mh(self):
+ # Convert MaildirMessage to MHMessage
+ msg_maildir = mailbox.MaildirMessage(_sample_message)
+ pairs = (('D', ['unseen']), ('F', ['unseen', 'flagged']),
+ ('P', ['unseen']), ('R', ['unseen', 'replied']), ('S', []),
+ ('T', ['unseen']), ('DFPRST', ['replied', 'flagged']))
+ for setting, result in pairs:
+ msg_maildir.set_flags(setting)
+ self.assertEqual(mailbox.MHMessage(msg_maildir).get_sequences(),
+ result)
+
+ def test_maildir_to_babyl(self):
+ # Convert MaildirMessage to Babyl
+ msg_maildir = mailbox.MaildirMessage(_sample_message)
+ pairs = (('D', ['unseen']), ('F', ['unseen']),
+ ('P', ['unseen', 'forwarded']), ('R', ['unseen', 'answered']),
+ ('S', []), ('T', ['unseen', 'deleted']),
+ ('DFPRST', ['deleted', 'answered', 'forwarded']))
+ for setting, result in pairs:
+ msg_maildir.set_flags(setting)
+ self.assertEqual(mailbox.BabylMessage(msg_maildir).get_labels(),
+ result)
+
+ def test_mboxmmdf_to_maildir(self):
+ # Convert mboxMessage and MMDFMessage to MaildirMessage
+ for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+ msg_mboxMMDF = class_(_sample_message)
+ msg_mboxMMDF.set_from('foo at bar', time.gmtime(0.0))
+ pairs = (('R', 'S'), ('O', ''), ('D', 'T'), ('F', 'F'), ('A', 'R'),
+ ('RODFA', 'FRST'))
+ for setting, result in pairs:
+ msg_mboxMMDF.set_flags(setting)
+ msg = mailbox.MaildirMessage(msg_mboxMMDF)
+ self.assertEqual(msg.get_flags(), result)
+ self.assertEqual(msg.get_date(), 0.0)
+ msg_mboxMMDF.set_flags('O')
+ self.assertEqual(mailbox.MaildirMessage(msg_mboxMMDF).get_subdir(),
+ 'cur')
+
+ def test_mboxmmdf_to_mboxmmdf(self):
+ # Convert mboxMessage and MMDFMessage to mboxMessage and MMDFMessage
+ for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+ msg_mboxMMDF = class_(_sample_message)
+ msg_mboxMMDF.set_flags('RODFA')
+ msg_mboxMMDF.set_from('foo at bar')
+ for class2_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+ msg2 = class2_(msg_mboxMMDF)
+ self.assertEqual(msg2.get_flags(), 'RODFA')
+ self.assertEqual(msg2.get_from(), 'foo at bar')
+
+ def test_mboxmmdf_to_mh(self):
+ # Convert mboxMessage and MMDFMessage to MHMessage
+ for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+ msg_mboxMMDF = class_(_sample_message)
+ pairs = (('R', []), ('O', ['unseen']), ('D', ['unseen']),
+ ('F', ['unseen', 'flagged']),
+ ('A', ['unseen', 'replied']),
+ ('RODFA', ['replied', 'flagged']))
+ for setting, result in pairs:
+ msg_mboxMMDF.set_flags(setting)
+ self.assertEqual(mailbox.MHMessage(msg_mboxMMDF).get_sequences(),
+ result)
+
+ def test_mboxmmdf_to_babyl(self):
+ # Convert mboxMessage and MMDFMessage to BabylMessage
+ for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+ msg = class_(_sample_message)
+ pairs = (('R', []), ('O', ['unseen']),
+ ('D', ['unseen', 'deleted']), ('F', ['unseen']),
+ ('A', ['unseen', 'answered']),
+ ('RODFA', ['deleted', 'answered']))
+ for setting, result in pairs:
+ msg.set_flags(setting)
+ self.assertEqual(mailbox.BabylMessage(msg).get_labels(), result)
+
+ def test_mh_to_maildir(self):
+ # Convert MHMessage to MaildirMessage
+ pairs = (('unseen', ''), ('replied', 'RS'), ('flagged', 'FS'))
+ for setting, result in pairs:
+ msg = mailbox.MHMessage(_sample_message)
+ msg.add_sequence(setting)
+ self.assertEqual(mailbox.MaildirMessage(msg).get_flags(), result)
+ self.assertEqual(mailbox.MaildirMessage(msg).get_subdir(), 'cur')
+ msg = mailbox.MHMessage(_sample_message)
+ msg.add_sequence('unseen')
+ msg.add_sequence('replied')
+ msg.add_sequence('flagged')
+ self.assertEqual(mailbox.MaildirMessage(msg).get_flags(), 'FR')
+ self.assertEqual(mailbox.MaildirMessage(msg).get_subdir(), 'cur')
+
+ def test_mh_to_mboxmmdf(self):
+ # Convert MHMessage to mboxMessage and MMDFMessage
+ pairs = (('unseen', 'O'), ('replied', 'ROA'), ('flagged', 'ROF'))
+ for setting, result in pairs:
+ msg = mailbox.MHMessage(_sample_message)
+ msg.add_sequence(setting)
+ for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+ self.assertEqual(class_(msg).get_flags(), result)
+ msg = mailbox.MHMessage(_sample_message)
+ msg.add_sequence('unseen')
+ msg.add_sequence('replied')
+ msg.add_sequence('flagged')
+ for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+ self.assertEqual(class_(msg).get_flags(), 'OFA')
+
+ def test_mh_to_mh(self):
+ # Convert MHMessage to MHMessage
+ msg = mailbox.MHMessage(_sample_message)
+ msg.add_sequence('unseen')
+ msg.add_sequence('replied')
+ msg.add_sequence('flagged')
+ self.assertEqual(mailbox.MHMessage(msg).get_sequences(),
+ ['unseen', 'replied', 'flagged'])
+
+ def test_mh_to_babyl(self):
+ # Convert MHMessage to BabylMessage
+ pairs = (('unseen', ['unseen']), ('replied', ['answered']),
+ ('flagged', []))
+ for setting, result in pairs:
+ msg = mailbox.MHMessage(_sample_message)
+ msg.add_sequence(setting)
+ self.assertEqual(mailbox.BabylMessage(msg).get_labels(), result)
+ msg = mailbox.MHMessage(_sample_message)
+ msg.add_sequence('unseen')
+ msg.add_sequence('replied')
+ msg.add_sequence('flagged')
+ self.assertEqual(mailbox.BabylMessage(msg).get_labels(),
+ ['unseen', 'answered'])
+
+ def test_babyl_to_maildir(self):
+ # Convert BabylMessage to MaildirMessage
+ pairs = (('unseen', ''), ('deleted', 'ST'), ('filed', 'S'),
+ ('answered', 'RS'), ('forwarded', 'PS'), ('edited', 'S'),
+ ('resent', 'PS'))
+ for setting, result in pairs:
+ msg = mailbox.BabylMessage(_sample_message)
+ msg.add_label(setting)
+ self.assertEqual(mailbox.MaildirMessage(msg).get_flags(), result)
+ self.assertEqual(mailbox.MaildirMessage(msg).get_subdir(), 'cur')
+ msg = mailbox.BabylMessage(_sample_message)
+ for label in ('unseen', 'deleted', 'filed', 'answered', 'forwarded',
+ 'edited', 'resent'):
+ msg.add_label(label)
+ self.assertEqual(mailbox.MaildirMessage(msg).get_flags(), 'PRT')
+ self.assertEqual(mailbox.MaildirMessage(msg).get_subdir(), 'cur')
+
+ def test_babyl_to_mboxmmdf(self):
+ # Convert BabylMessage to mboxMessage and MMDFMessage
+ pairs = (('unseen', 'O'), ('deleted', 'ROD'), ('filed', 'RO'),
+ ('answered', 'ROA'), ('forwarded', 'RO'), ('edited', 'RO'),
+ ('resent', 'RO'))
+ for setting, result in pairs:
+ for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+ msg = mailbox.BabylMessage(_sample_message)
+ msg.add_label(setting)
+ self.assertEqual(class_(msg).get_flags(), result)
+ msg = mailbox.BabylMessage(_sample_message)
+ for label in ('unseen', 'deleted', 'filed', 'answered', 'forwarded',
+ 'edited', 'resent'):
+ msg.add_label(label)
+ for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+ self.assertEqual(class_(msg).get_flags(), 'ODA')
+
+ def test_babyl_to_mh(self):
+ # Convert BabylMessage to MHMessage
+ pairs = (('unseen', ['unseen']), ('deleted', []), ('filed', []),
+ ('answered', ['replied']), ('forwarded', []), ('edited', []),
+ ('resent', []))
+ for setting, result in pairs:
+ msg = mailbox.BabylMessage(_sample_message)
+ msg.add_label(setting)
+ self.assertEqual(mailbox.MHMessage(msg).get_sequences(), result)
+ msg = mailbox.BabylMessage(_sample_message)
+ for label in ('unseen', 'deleted', 'filed', 'answered', 'forwarded',
+ 'edited', 'resent'):
+ msg.add_label(label)
+ self.assertEqual(mailbox.MHMessage(msg).get_sequences(),
+ ['unseen', 'replied'])
+
+ def test_babyl_to_babyl(self):
+ # Convert BabylMessage to BabylMessage
+ msg = mailbox.BabylMessage(_sample_message)
+ msg.update_visible()
+ for label in ('unseen', 'deleted', 'filed', 'answered', 'forwarded',
+ 'edited', 'resent'):
+ msg.add_label(label)
+ msg2 = mailbox.BabylMessage(msg)
+ self.assertEqual(msg2.get_labels(), ['unseen', 'deleted', 'filed',
+ 'answered', 'forwarded', 'edited',
+ 'resent'])
+ self.assertEqual(msg.get_visible().keys(), msg2.get_visible().keys())
+ for key in msg.get_visible().keys():
+ self.assertEqual(msg.get_visible()[key], msg2.get_visible()[key])
+
+
+class TestProxyFileBase(TestBase):
+
+ def _test_read(self, proxy):
+ # Read by byte
+ proxy.seek(0)
+ self.assertEqual(proxy.read(), 'bar')
+ proxy.seek(1)
+ self.assertEqual(proxy.read(), 'ar')
+ proxy.seek(0)
+ self.assertEqual(proxy.read(2), 'ba')
+ proxy.seek(1)
+ self.assertEqual(proxy.read(-1), 'ar')
+ proxy.seek(2)
+ self.assertEqual(proxy.read(1000), 'r')
+
+ def _test_readline(self, proxy):
+ # Read by line
+ proxy.seek(0)
+ self.assertEqual(proxy.readline(), 'foo' + os.linesep)
+ self.assertEqual(proxy.readline(), 'bar' + os.linesep)
+ self.assertEqual(proxy.readline(), 'fred' + os.linesep)
+ self.assertEqual(proxy.readline(), 'bob')
+ proxy.seek(2)
+ self.assertEqual(proxy.readline(), 'o' + os.linesep)
+ proxy.seek(6 + 2 * len(os.linesep))
+ self.assertEqual(proxy.readline(), 'fred' + os.linesep)
+ proxy.seek(6 + 2 * len(os.linesep))
+ self.assertEqual(proxy.readline(2), 'fr')
+ self.assertEqual(proxy.readline(-10), 'ed' + os.linesep)
+
+ def _test_readlines(self, proxy):
+ # Read multiple lines
+ proxy.seek(0)
+ self.assertEqual(proxy.readlines(), ['foo' + os.linesep,
+ 'bar' + os.linesep,
+ 'fred' + os.linesep, 'bob'])
+ proxy.seek(0)
+ self.assertEqual(proxy.readlines(2), ['foo' + os.linesep])
+ proxy.seek(3 + len(os.linesep))
+ self.assertEqual(proxy.readlines(4 + len(os.linesep)),
+ ['bar' + os.linesep, 'fred' + os.linesep])
+ proxy.seek(3)
+ self.assertEqual(proxy.readlines(1000), [os.linesep, 'bar' + os.linesep,
+ 'fred' + os.linesep, 'bob'])
+
+ def _test_iteration(self, proxy):
+ # Iterate by line
+ proxy.seek(0)
+ iterator = iter(proxy)
+ self.assertEqual(list(iterator),
+ ['foo' + os.linesep, 'bar' + os.linesep, 'fred' + os.linesep, 'bob'])
+
+ def _test_seek_and_tell(self, proxy):
+ # Seek and use tell to check position
+ proxy.seek(3)
+ self.assertEqual(proxy.tell(), 3)
+ self.assertEqual(proxy.read(len(os.linesep)), os.linesep)
+ proxy.seek(2, 1)
+ self.assertEqual(proxy.read(1 + len(os.linesep)), 'r' + os.linesep)
+ proxy.seek(-3 - len(os.linesep), 2)
+ self.assertEqual(proxy.read(3), 'bar')
+ proxy.seek(2, 0)
+ self.assertEqual(proxy.read(), 'o' + os.linesep + 'bar' + os.linesep)
+ proxy.seek(100)
+ self.assertEqual(proxy.read(), '')
+
+ def _test_close(self, proxy):
+ # Close a file
+ proxy.close()
+ self.assertRaises(AttributeError, lambda: proxy.close())
+
+
+class TestProxyFile(TestProxyFileBase):
+
+ def setUp(self):
+ self._path = test_support.TESTFN
+ self._file = open(self._path, 'wb+')
+
+ def tearDown(self):
+ self._file.close()
+ self._delete_recursively(self._path)
+
+ def test_initialize(self):
+ # Initialize and check position
+ self._file.write('foo')
+ pos = self._file.tell()
+ proxy0 = mailbox._ProxyFile(self._file)
+ self.assertEqual(proxy0.tell(), pos)
+ self.assertEqual(self._file.tell(), pos)
+ proxy1 = mailbox._ProxyFile(self._file, 0)
+ self.assertEqual(proxy1.tell(), 0)
+ self.assertEqual(self._file.tell(), pos)
+
+ def test_read(self):
+ self._file.write('bar')
+ self._test_read(mailbox._ProxyFile(self._file))
+
+ def test_readline(self):
+ self._file.write('foo%sbar%sfred%sbob' % (os.linesep, os.linesep,
+ os.linesep))
+ self._test_readline(mailbox._ProxyFile(self._file))
+
+ def test_readlines(self):
+ self._file.write('foo%sbar%sfred%sbob' % (os.linesep, os.linesep,
+ os.linesep))
+ self._test_readlines(mailbox._ProxyFile(self._file))
+
+ def test_iteration(self):
+ self._file.write('foo%sbar%sfred%sbob' % (os.linesep, os.linesep,
+ os.linesep))
+ self._test_iteration(mailbox._ProxyFile(self._file))
+
+ def test_seek_and_tell(self):
+ self._file.write('foo%sbar%s' % (os.linesep, os.linesep))
+ self._test_seek_and_tell(mailbox._ProxyFile(self._file))
+
+ def test_close(self):
+ self._file.write('foo%sbar%s' % (os.linesep, os.linesep))
+ self._test_close(mailbox._ProxyFile(self._file))
+
+
+class TestPartialFile(TestProxyFileBase):
+
+ def setUp(self):
+ self._path = test_support.TESTFN
+ self._file = open(self._path, 'wb+')
+
+ def tearDown(self):
+ self._file.close()
+ self._delete_recursively(self._path)
+
+ def test_initialize(self):
+ # Initialize and check position
+ self._file.write('foo' + os.linesep + 'bar')
+ pos = self._file.tell()
+ proxy = mailbox._PartialFile(self._file, 2, 5)
+ self.assertEqual(proxy.tell(), 0)
+ self.assertEqual(self._file.tell(), pos)
+
+ def test_read(self):
+ self._file.write('***bar***')
+ self._test_read(mailbox._PartialFile(self._file, 3, 6))
+
+ def test_readline(self):
+ self._file.write('!!!!!foo%sbar%sfred%sbob!!!!!' %
+ (os.linesep, os.linesep, os.linesep))
+ self._test_readline(mailbox._PartialFile(self._file, 5,
+ 18 + 3 * len(os.linesep)))
+
+ def test_readlines(self):
+ self._file.write('foo%sbar%sfred%sbob?????' %
+ (os.linesep, os.linesep, os.linesep))
+ self._test_readlines(mailbox._PartialFile(self._file, 0,
+ 13 + 3 * len(os.linesep)))
+
+ def test_iteration(self):
+ self._file.write('____foo%sbar%sfred%sbob####' %
+ (os.linesep, os.linesep, os.linesep))
+ self._test_iteration(mailbox._PartialFile(self._file, 4,
+ 17 + 3 * len(os.linesep)))
+
+ def test_seek_and_tell(self):
+ self._file.write('(((foo%sbar%s$$$' % (os.linesep, os.linesep))
+ self._test_seek_and_tell(mailbox._PartialFile(self._file, 3,
+ 9 + 2 * len(os.linesep)))
+
+ def test_close(self):
+ self._file.write('&foo%sbar%s^' % (os.linesep, os.linesep))
+ self._test_close(mailbox._PartialFile(self._file, 1,
+ 6 + 3 * len(os.linesep)))
+
+
+## Start: tests from the original module (for backward compatibility).
+
+FROM_ = "From some.body at dummy.domain Sat Jul 24 13:43:35 2004\n"
+DUMMY_MESSAGE = """\
+From: some.body at dummy.domain
+To: me at my.domain
+Subject: Simple Test
+
+This is a dummy message.
+"""
+
+class MaildirTestCase(unittest.TestCase):
+
+ def setUp(self):
+ # create a new maildir mailbox to work with:
+ self._dir = test_support.TESTFN
+ os.mkdir(self._dir)
+ os.mkdir(os.path.join(self._dir, "cur"))
+ os.mkdir(os.path.join(self._dir, "tmp"))
+ os.mkdir(os.path.join(self._dir, "new"))
+ self._counter = 1
+ self._msgfiles = []
+
+ def tearDown(self):
+ map(os.unlink, self._msgfiles)
+ os.rmdir(os.path.join(self._dir, "cur"))
+ os.rmdir(os.path.join(self._dir, "tmp"))
+ os.rmdir(os.path.join(self._dir, "new"))
+ os.rmdir(self._dir)
+
+ def createMessage(self, dir, mbox=False):
+ t = int(time.time() % 1000000)
+ pid = self._counter
+ self._counter += 1
+ filename = os.extsep.join((str(t), str(pid), "myhostname", "mydomain"))
+ tmpname = os.path.join(self._dir, "tmp", filename)
+ newname = os.path.join(self._dir, dir, filename)
+ with open(tmpname, "w") as fp:
+ self._msgfiles.append(tmpname)
+ if mbox:
+ fp.write(FROM_)
+ fp.write(DUMMY_MESSAGE)
+ if hasattr(os, "link"):
+ os.link(tmpname, newname)
+ else:
+ with open(newname, "w") as fp:
+ fp.write(DUMMY_MESSAGE)
+ self._msgfiles.append(newname)
+ return tmpname
+
+ def test_empty_maildir(self):
+ """Test an empty maildir mailbox"""
+ # Test for regression on bug #117490:
+ # Make sure the boxes attribute actually gets set.
+ self.mbox = mailbox.Maildir(test_support.TESTFN)
+ #self.assertTrue(hasattr(self.mbox, "boxes"))
+ #self.assertTrue(len(self.mbox.boxes) == 0)
+ self.assertIs(self.mbox.next(), None)
+ self.assertIs(self.mbox.next(), None)
+
+ def test_nonempty_maildir_cur(self):
+ self.createMessage("cur")
+ self.mbox = mailbox.Maildir(test_support.TESTFN)
+ #self.assertTrue(len(self.mbox.boxes) == 1)
+ self.assertIsNot(self.mbox.next(), None)
+ self.assertIs(self.mbox.next(), None)
+ self.assertIs(self.mbox.next(), None)
+
+ def test_nonempty_maildir_new(self):
+ self.createMessage("new")
+ self.mbox = mailbox.Maildir(test_support.TESTFN)
+ #self.assertTrue(len(self.mbox.boxes) == 1)
+ self.assertIsNot(self.mbox.next(), None)
+ self.assertIs(self.mbox.next(), None)
+ self.assertIs(self.mbox.next(), None)
+
+ def test_nonempty_maildir_both(self):
+ self.createMessage("cur")
+ self.createMessage("new")
+ self.mbox = mailbox.Maildir(test_support.TESTFN)
+ #self.assertTrue(len(self.mbox.boxes) == 2)
+ self.assertIsNot(self.mbox.next(), None)
+ self.assertIsNot(self.mbox.next(), None)
+ self.assertIs(self.mbox.next(), None)
+ self.assertIs(self.mbox.next(), None)
+
+ def test_unix_mbox(self):
+ ### should be better!
+ import email.parser
+ fname = self.createMessage("cur", True)
+ n = 0
+ for msg in mailbox.PortableUnixMailbox(open(fname),
+ email.parser.Parser().parse):
+ n += 1
+ self.assertEqual(msg["subject"], "Simple Test")
+ self.assertEqual(len(str(msg)), len(FROM_)+len(DUMMY_MESSAGE))
+ self.assertEqual(n, 1)
+
+## End: classes from the original module (for backward compatibility).
+
+
+_sample_message = """\
+Return-Path: <gkj at gregorykjohnson.com>
+X-Original-To: gkj+person at localhost
+Delivered-To: gkj+person at localhost
+Received: from localhost (localhost [127.0.0.1])
+ by andy.gregorykjohnson.com (Postfix) with ESMTP id 356ED9DD17
+ for <gkj+person at localhost>; Wed, 13 Jul 2005 17:23:16 -0400 (EDT)
+Delivered-To: gkj at sundance.gregorykjohnson.com
+Received: from localhost [127.0.0.1]
+ by localhost with POP3 (fetchmail-6.2.5)
+ for gkj+person at localhost (single-drop); Wed, 13 Jul 2005 17:23:16 -0400 (EDT)
+Received: from andy.gregorykjohnson.com (andy.gregorykjohnson.com [64.32.235.228])
+ by sundance.gregorykjohnson.com (Postfix) with ESMTP id 5B056316746
+ for <gkj at gregorykjohnson.com>; Wed, 13 Jul 2005 17:23:11 -0400 (EDT)
+Received: by andy.gregorykjohnson.com (Postfix, from userid 1000)
+ id 490CD9DD17; Wed, 13 Jul 2005 17:23:11 -0400 (EDT)
+Date: Wed, 13 Jul 2005 17:23:11 -0400
+From: "Gregory K. Johnson" <gkj at gregorykjohnson.com>
+To: gkj at gregorykjohnson.com
+Subject: Sample message
+Message-ID: <20050713212311.GC4701 at andy.gregorykjohnson.com>
+Mime-Version: 1.0
+Content-Type: multipart/mixed; boundary="NMuMz9nt05w80d4+"
+Content-Disposition: inline
+User-Agent: Mutt/1.5.9i
+
+
+--NMuMz9nt05w80d4+
+Content-Type: text/plain; charset=us-ascii
+Content-Disposition: inline
+
+This is a sample message.
+
+--
+Gregory K. Johnson
+
+--NMuMz9nt05w80d4+
+Content-Type: application/octet-stream
+Content-Disposition: attachment; filename="text.gz"
+Content-Transfer-Encoding: base64
+
+H4sICM2D1UIAA3RleHQAC8nILFYAokSFktSKEoW0zJxUPa7wzJIMhZLyfIWczLzUYj0uAHTs
+3FYlAAAA
+
+--NMuMz9nt05w80d4+--
+"""
+
+_sample_headers = {
+ "Return-Path":"<gkj at gregorykjohnson.com>",
+ "X-Original-To":"gkj+person at localhost",
+ "Delivered-To":"gkj+person at localhost",
+ "Received":"""from localhost (localhost [127.0.0.1])
+ by andy.gregorykjohnson.com (Postfix) with ESMTP id 356ED9DD17
+ for <gkj+person at localhost>; Wed, 13 Jul 2005 17:23:16 -0400 (EDT)""",
+ "Delivered-To":"gkj at sundance.gregorykjohnson.com",
+ "Received":"""from localhost [127.0.0.1]
+ by localhost with POP3 (fetchmail-6.2.5)
+ for gkj+person at localhost (single-drop); Wed, 13 Jul 2005 17:23:16 -0400 (EDT)""",
+ "Received":"""from andy.gregorykjohnson.com (andy.gregorykjohnson.com [64.32.235.228])
+ by sundance.gregorykjohnson.com (Postfix) with ESMTP id 5B056316746
+ for <gkj at gregorykjohnson.com>; Wed, 13 Jul 2005 17:23:11 -0400 (EDT)""",
+ "Received":"""by andy.gregorykjohnson.com (Postfix, from userid 1000)
+ id 490CD9DD17; Wed, 13 Jul 2005 17:23:11 -0400 (EDT)""",
+ "Date":"Wed, 13 Jul 2005 17:23:11 -0400",
+ "From":""""Gregory K. Johnson" <gkj at gregorykjohnson.com>""",
+ "To":"gkj at gregorykjohnson.com",
+ "Subject":"Sample message",
+ "Mime-Version":"1.0",
+ "Content-Type":"""multipart/mixed; boundary="NMuMz9nt05w80d4+\"""",
+ "Content-Disposition":"inline",
+ "User-Agent": "Mutt/1.5.9i" }
+
+_sample_payloads = ("""This is a sample message.
+
+--
+Gregory K. Johnson
+""",
+"""H4sICM2D1UIAA3RleHQAC8nILFYAokSFktSKEoW0zJxUPa7wzJIMhZLyfIWczLzUYj0uAHTs
+3FYlAAAA
+""")
+
+
+def test_main():
+ tests = (TestMailboxSuperclass, TestMaildir, TestMbox, TestMMDF, TestMH,
+ TestBabyl, TestMessage, TestMaildirMessage, TestMboxMessage,
+ TestMHMessage, TestBabylMessage, TestMMDFMessage,
+ TestMessageConversion, TestProxyFile, TestPartialFile,
+ MaildirTestCase)
+ test_support.run_unittest(*tests)
+ test_support.reap_children()
+
+
+if __name__ == '__main__':
+ test_main()
diff --git a/lib-python/2.7/test/test_old_mailbox.py b/lib-python/2.7/test/test_old_mailbox.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/test_old_mailbox.py
@@ -0,0 +1,152 @@
+# This set of tests exercises the backward-compatibility class
+# in mailbox.py (the ones without write support).
+
+import mailbox
+import os
+import time
+import unittest
+from test import test_support
+
+# cleanup earlier tests
+try:
+ os.unlink(test_support.TESTFN)
+except os.error:
+ pass
+
+FROM_ = "From some.body at dummy.domain Sat Jul 24 13:43:35 2004\n"
+DUMMY_MESSAGE = """\
+From: some.body at dummy.domain
+To: me at my.domain
+Subject: Simple Test
+
+This is a dummy message.
+"""
+
+class MaildirTestCase(unittest.TestCase):
+
+ def setUp(self):
+ # create a new maildir mailbox to work with:
+ self._dir = test_support.TESTFN
+ os.mkdir(self._dir)
+ os.mkdir(os.path.join(self._dir, "cur"))
+ os.mkdir(os.path.join(self._dir, "tmp"))
+ os.mkdir(os.path.join(self._dir, "new"))
+ self._counter = 1
+ self._msgfiles = []
+
+ def tearDown(self):
+ map(os.unlink, self._msgfiles)
+ os.rmdir(os.path.join(self._dir, "cur"))
+ os.rmdir(os.path.join(self._dir, "tmp"))
+ os.rmdir(os.path.join(self._dir, "new"))
+ os.rmdir(self._dir)
+
+ def createMessage(self, dir, mbox=False):
+ t = int(time.time() % 1000000)
+ pid = self._counter
+ self._counter += 1
+ filename = os.extsep.join((str(t), str(pid), "myhostname", "mydomain"))
+ tmpname = os.path.join(self._dir, "tmp", filename)
+ newname = os.path.join(self._dir, dir, filename)
+ with open(tmpname, "w") as fp:
+ self._msgfiles.append(tmpname)
+ if mbox:
+ fp.write(FROM_)
+ fp.write(DUMMY_MESSAGE)
+ if hasattr(os, "link"):
+ os.link(tmpname, newname)
+ else:
+ with open(newname, "w") as fp:
+ fp.write(DUMMY_MESSAGE)
+ self._msgfiles.append(newname)
+ return tmpname
+
+ def test_empty_maildir(self):
+ """Test an empty maildir mailbox"""
+ # Test for regression on bug #117490:
+ self.mbox = mailbox.Maildir(test_support.TESTFN)
+ self.assertTrue(len(self.mbox) == 0)
+ self.assertTrue(self.mbox.next() is None)
+ self.assertTrue(self.mbox.next() is None)
+
+ def test_nonempty_maildir_cur(self):
+ self.createMessage("cur")
+ self.mbox = mailbox.Maildir(test_support.TESTFN)
+ self.assertTrue(len(self.mbox) == 1)
+ self.assertTrue(self.mbox.next() is not None)
+ self.assertTrue(self.mbox.next() is None)
+ self.assertTrue(self.mbox.next() is None)
+
+ def test_nonempty_maildir_new(self):
+ self.createMessage("new")
+ self.mbox = mailbox.Maildir(test_support.TESTFN)
+ self.assertTrue(len(self.mbox) == 1)
+ self.assertTrue(self.mbox.next() is not None)
+ self.assertTrue(self.mbox.next() is None)
+ self.assertTrue(self.mbox.next() is None)
+
+ def test_nonempty_maildir_both(self):
+ self.createMessage("cur")
+ self.createMessage("new")
+ self.mbox = mailbox.Maildir(test_support.TESTFN)
+ self.assertTrue(len(self.mbox) == 2)
+ self.assertTrue(self.mbox.next() is not None)
+ self.assertTrue(self.mbox.next() is not None)
+ self.assertTrue(self.mbox.next() is None)
+ self.assertTrue(self.mbox.next() is None)
+
+ def test_unix_mbox(self):
+ ### should be better!
+ import email.parser
+ fname = self.createMessage("cur", True)
+ n = 0
+ with open(fname) as f:
+ for msg in mailbox.PortableUnixMailbox(f,
+ email.parser.Parser().parse):
+ n += 1
+ self.assertEqual(msg["subject"], "Simple Test")
+ self.assertEqual(len(str(msg)), len(FROM_)+len(DUMMY_MESSAGE))
+ self.assertEqual(n, 1)
+
+class MboxTestCase(unittest.TestCase):
+ def setUp(self):
+ # create a new maildir mailbox to work with:
+ self._path = test_support.TESTFN
+
+ def tearDown(self):
+ os.unlink(self._path)
+
+ def test_from_regex (self):
+ # Testing new regex from bug #1633678
+ with open(self._path, 'w') as f:
+ f.write("""From fred at example.com Mon May 31 13:24:50 2004 +0200
+Subject: message 1
+
+body1
+From fred at example.com Mon May 31 13:24:50 2004 -0200
+Subject: message 2
+
+body2
+From fred at example.com Mon May 31 13:24:50 2004
+Subject: message 3
+
+body3
+From fred at example.com Mon May 31 13:24:50 2004
+Subject: message 4
+
+body4
+""")
+ with open(self._path, 'r') as f:
+ box = mailbox.UnixMailbox(f)
+ self.assertTrue(len(list(iter(box))) == 4)
+
+
+ # XXX We still need more tests!
+
+
+def test_main():
+ test_support.run_unittest(MaildirTestCase, MboxTestCase)
+
+
+if __name__ == "__main__":
+ test_main()
diff --git a/lib-python/2.7/test/test_os.py b/lib-python/2.7/test/test_os.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/test_os.py
@@ -0,0 +1,824 @@
+# As a test suite for the os module, this is woefully inadequate, but this
+# does add tests for a few functions which have been determined to be more
+# portable than they had been thought to be.
+
+import os
+import errno
+import unittest
+import warnings
+import sys
+import signal
+import subprocess
+import time
+from test import test_support
+import mmap
+import uuid
+
+warnings.filterwarnings("ignore", "tempnam", RuntimeWarning, __name__)
+warnings.filterwarnings("ignore", "tmpnam", RuntimeWarning, __name__)
+
+# Tests creating TESTFN
+class FileTests(unittest.TestCase):
+ def setUp(self):
+ if os.path.exists(test_support.TESTFN):
+ os.unlink(test_support.TESTFN)
+ tearDown = setUp
+
+ def test_access(self):
+ f = os.open(test_support.TESTFN, os.O_CREAT|os.O_RDWR)
+ os.close(f)
+ self.assertTrue(os.access(test_support.TESTFN, os.W_OK))
+
+ def test_closerange(self):
+ first = os.open(test_support.TESTFN, os.O_CREAT|os.O_RDWR)
+ # We must allocate two consecutive file descriptors, otherwise
+ # it will mess up other file descriptors (perhaps even the three
+ # standard ones).
+ second = os.dup(first)
+ try:
+ retries = 0
+ while second != first + 1:
+ os.close(first)
+ retries += 1
+ if retries > 10:
+ # XXX test skipped
+ self.skipTest("couldn't allocate two consecutive fds")
+ first, second = second, os.dup(second)
+ finally:
+ os.close(second)
+ # close a fd that is open, and one that isn't
+ os.closerange(first, first + 2)
+ self.assertRaises(OSError, os.write, first, "a")
+
+ @test_support.cpython_only
+ def test_rename(self):
+ path = unicode(test_support.TESTFN)
+ old = sys.getrefcount(path)
+ self.assertRaises(TypeError, os.rename, path, 0)
+ new = sys.getrefcount(path)
+ self.assertEqual(old, new)
+
+
+class TemporaryFileTests(unittest.TestCase):
+ def setUp(self):
+ self.files = []
+ os.mkdir(test_support.TESTFN)
+
+ def tearDown(self):
+ for name in self.files:
+ os.unlink(name)
+ os.rmdir(test_support.TESTFN)
+
+ def check_tempfile(self, name):
+ # make sure it doesn't already exist:
+ self.assertFalse(os.path.exists(name),
+ "file already exists for temporary file")
+ # make sure we can create the file
+ open(name, "w")
+ self.files.append(name)
+
+ def test_tempnam(self):
+ if not hasattr(os, "tempnam"):
+ return
+ with warnings.catch_warnings():
+ warnings.filterwarnings("ignore", "tempnam", RuntimeWarning,
+ r"test_os$")
+ warnings.filterwarnings("ignore", "tempnam", DeprecationWarning)
+ self.check_tempfile(os.tempnam())
+
+ name = os.tempnam(test_support.TESTFN)
+ self.check_tempfile(name)
+
+ name = os.tempnam(test_support.TESTFN, "pfx")
+ self.assertTrue(os.path.basename(name)[:3] == "pfx")
+ self.check_tempfile(name)
+
+ def test_tmpfile(self):
+ if not hasattr(os, "tmpfile"):
+ return
+ # As with test_tmpnam() below, the Windows implementation of tmpfile()
+ # attempts to create a file in the root directory of the current drive.
+ # On Vista and Server 2008, this test will always fail for normal users
+ # as writing to the root directory requires elevated privileges. With
+ # XP and below, the semantics of tmpfile() are the same, but the user
+ # running the test is more likely to have administrative privileges on
+ # their account already. If that's the case, then os.tmpfile() should
+ # work. In order to make this test as useful as possible, rather than
+ # trying to detect Windows versions or whether or not the user has the
+ # right permissions, just try and create a file in the root directory
+ # and see if it raises a 'Permission denied' OSError. If it does, then
+ # test that a subsequent call to os.tmpfile() raises the same error. If
+ # it doesn't, assume we're on XP or below and the user running the test
+ # has administrative privileges, and proceed with the test as normal.
+ with warnings.catch_warnings():
+ warnings.filterwarnings("ignore", "tmpfile", DeprecationWarning)
+
+ if sys.platform == 'win32':
+ name = '\\python_test_os_test_tmpfile.txt'
+ if os.path.exists(name):
+ os.remove(name)
+ try:
+ fp = open(name, 'w')
+ except IOError, first:
+ # open() failed, assert tmpfile() fails in the same way.
+ # Although open() raises an IOError and os.tmpfile() raises an
+ # OSError(), 'args' will be (13, 'Permission denied') in both
+ # cases.
+ try:
+ fp = os.tmpfile()
+ except OSError, second:
+ self.assertEqual(first.args, second.args)
+ else:
+ self.fail("expected os.tmpfile() to raise OSError")
+ return
+ else:
+ # open() worked, therefore, tmpfile() should work. Close our
+ # dummy file and proceed with the test as normal.
+ fp.close()
+ os.remove(name)
+
+ fp = os.tmpfile()
+ fp.write("foobar")
+ fp.seek(0,0)
+ s = fp.read()
+ fp.close()
+ self.assertTrue(s == "foobar")
+
+ def test_tmpnam(self):
+ if not hasattr(os, "tmpnam"):
+ return
+ with warnings.catch_warnings():
+ warnings.filterwarnings("ignore", "tmpnam", RuntimeWarning,
+ r"test_os$")
+ warnings.filterwarnings("ignore", "tmpnam", DeprecationWarning)
+
+ name = os.tmpnam()
+ if sys.platform in ("win32",):
+ # The Windows tmpnam() seems useless. From the MS docs:
+ #
+ # The character string that tmpnam creates consists of
+ # the path prefix, defined by the entry P_tmpdir in the
+ # file STDIO.H, followed by a sequence consisting of the
+ # digit characters '0' through '9'; the numerical value
+ # of this string is in the range 1 - 65,535. Changing the
+ # definitions of L_tmpnam or P_tmpdir in STDIO.H does not
+ # change the operation of tmpnam.
+ #
+ # The really bizarre part is that, at least under MSVC6,
+ # P_tmpdir is "\\". That is, the path returned refers to
+ # the root of the current drive. That's a terrible place to
+ # put temp files, and, depending on privileges, the user
+ # may not even be able to open a file in the root directory.
+ self.assertFalse(os.path.exists(name),
+ "file already exists for temporary file")
+ else:
+ self.check_tempfile(name)
+
+# Test attributes on return values from os.*stat* family.
+class StatAttributeTests(unittest.TestCase):
+ def setUp(self):
+ os.mkdir(test_support.TESTFN)
+ self.fname = os.path.join(test_support.TESTFN, "f1")
+ f = open(self.fname, 'wb')
+ f.write("ABC")
+ f.close()
+
+ def tearDown(self):
+ os.unlink(self.fname)
+ os.rmdir(test_support.TESTFN)
+
+ def test_stat_attributes(self):
+ if not hasattr(os, "stat"):
+ return
+
+ import stat
+ result = os.stat(self.fname)
+
+ # Make sure direct access works
+ self.assertEqual(result[stat.ST_SIZE], 3)
+ self.assertEqual(result.st_size, 3)
+
+ # Make sure all the attributes are there
+ members = dir(result)
+ for name in dir(stat):
+ if name[:3] == 'ST_':
+ attr = name.lower()
+ if name.endswith("TIME"):
+ def trunc(x): return int(x)
+ else:
+ def trunc(x): return x
+ self.assertEqual(trunc(getattr(result, attr)),
+ result[getattr(stat, name)])
+ self.assertIn(attr, members)
+
+ try:
+ result[200]
+ self.fail("No exception thrown")
+ except IndexError:
+ pass
+
+ # Make sure that assignment fails
+ try:
+ result.st_mode = 1
+ self.fail("No exception thrown")
+ except (AttributeError, TypeError):
+ pass
+
+ try:
+ result.st_rdev = 1
+ self.fail("No exception thrown")
+ except (AttributeError, TypeError):
+ pass
+
+ try:
+ result.parrot = 1
+ self.fail("No exception thrown")
+ except AttributeError:
+ pass
+
+ # Use the stat_result constructor with a too-short tuple.
+ try:
+ result2 = os.stat_result((10,))
+ self.fail("No exception thrown")
+ except TypeError:
+ pass
+
+ # Use the constructor with a too-long tuple.
+ try:
+ result2 = os.stat_result((0,1,2,3,4,5,6,7,8,9,10,11,12,13,14))
+ except TypeError:
+ pass
+
+
+ def test_statvfs_attributes(self):
+ if not hasattr(os, "statvfs"):
+ return
+
+ try:
+ result = os.statvfs(self.fname)
+ except OSError, e:
+ # On AtheOS, glibc always returns ENOSYS
+ if e.errno == errno.ENOSYS:
+ return
+
+ # Make sure direct access works
+ self.assertEqual(result.f_bfree, result[3])
+
+ # Make sure all the attributes are there.
+ members = ('bsize', 'frsize', 'blocks', 'bfree', 'bavail', 'files',
+ 'ffree', 'favail', 'flag', 'namemax')
+ for value, member in enumerate(members):
+ self.assertEqual(getattr(result, 'f_' + member), result[value])
+
+ # Make sure that assignment really fails
+ try:
+ result.f_bfree = 1
+ self.fail("No exception thrown")
+ except TypeError:
+ pass
+
+ try:
+ result.parrot = 1
+ self.fail("No exception thrown")
+ except AttributeError:
+ pass
+
+ # Use the constructor with a too-short tuple.
+ try:
+ result2 = os.statvfs_result((10,))
+ self.fail("No exception thrown")
+ except TypeError:
+ pass
+
+ # Use the constructor with a too-long tuple.
+ try:
+ result2 = os.statvfs_result((0,1,2,3,4,5,6,7,8,9,10,11,12,13,14))
+ except TypeError:
+ pass
+
+ def test_utime_dir(self):
+ delta = 1000000
+ st = os.stat(test_support.TESTFN)
+ # round to int, because some systems may support sub-second
+ # time stamps in stat, but not in utime.
+ os.utime(test_support.TESTFN, (st.st_atime, int(st.st_mtime-delta)))
+ st2 = os.stat(test_support.TESTFN)
+ self.assertEqual(st2.st_mtime, int(st.st_mtime-delta))
+
+ # Restrict test to Win32, since there is no guarantee other
+ # systems support centiseconds
+ if sys.platform == 'win32':
+ def get_file_system(path):
+ root = os.path.splitdrive(os.path.abspath(path))[0] + '\\'
+ import ctypes
+ kernel32 = ctypes.windll.kernel32
+ buf = ctypes.create_string_buffer("", 100)
+ if kernel32.GetVolumeInformationA(root, None, 0, None, None, None, buf, len(buf)):
+ return buf.value
+
+ if get_file_system(test_support.TESTFN) == "NTFS":
+ def test_1565150(self):
+ t1 = 1159195039.25
+ os.utime(self.fname, (t1, t1))
+ self.assertEqual(os.stat(self.fname).st_mtime, t1)
+
+ def test_large_time(self):
+ t1 = 5000000000 # some day in 2128
+ os.utime(self.fname, (t1, t1))
+ self.assertEqual(os.stat(self.fname).st_mtime, t1)
+
+ def test_1686475(self):
+ # Verify that an open file can be stat'ed
+ try:
+ os.stat(r"c:\pagefile.sys")
+ except WindowsError, e:
+ if e.errno == 2: # file does not exist; cannot run test
+ return
+ self.fail("Could not stat pagefile.sys")
+
+from test import mapping_tests
+
+class EnvironTests(mapping_tests.BasicTestMappingProtocol):
+ """check that os.environ object conform to mapping protocol"""
+ type2test = None
+ def _reference(self):
+ return {"KEY1":"VALUE1", "KEY2":"VALUE2", "KEY3":"VALUE3"}
+ def _empty_mapping(self):
+ os.environ.clear()
+ return os.environ
+ def setUp(self):
+ self.__save = dict(os.environ)
+ os.environ.clear()
+ def tearDown(self):
+ os.environ.clear()
+ os.environ.update(self.__save)
+
+ # Bug 1110478
+ def test_update2(self):
+ if os.path.exists("/bin/sh"):
+ os.environ.update(HELLO="World")
+ with os.popen("/bin/sh -c 'echo $HELLO'") as popen:
+ value = popen.read().strip()
+ self.assertEqual(value, "World")
+
+class WalkTests(unittest.TestCase):
+ """Tests for os.walk()."""
+
+ def test_traversal(self):
+ import os
+ from os.path import join
+
+ # Build:
+ # TESTFN/
+ # TEST1/ a file kid and two directory kids
+ # tmp1
+ # SUB1/ a file kid and a directory kid
+ # tmp2
+ # SUB11/ no kids
+ # SUB2/ a file kid and a dirsymlink kid
+ # tmp3
+ # link/ a symlink to TESTFN.2
+ # TEST2/
+ # tmp4 a lone file
+ walk_path = join(test_support.TESTFN, "TEST1")
+ sub1_path = join(walk_path, "SUB1")
+ sub11_path = join(sub1_path, "SUB11")
+ sub2_path = join(walk_path, "SUB2")
+ tmp1_path = join(walk_path, "tmp1")
+ tmp2_path = join(sub1_path, "tmp2")
+ tmp3_path = join(sub2_path, "tmp3")
+ link_path = join(sub2_path, "link")
+ t2_path = join(test_support.TESTFN, "TEST2")
+ tmp4_path = join(test_support.TESTFN, "TEST2", "tmp4")
+
+ # Create stuff.
+ os.makedirs(sub11_path)
+ os.makedirs(sub2_path)
+ os.makedirs(t2_path)
+ for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path:
+ f = file(path, "w")
+ f.write("I'm " + path + " and proud of it. Blame test_os.\n")
+ f.close()
+ if hasattr(os, "symlink"):
+ os.symlink(os.path.abspath(t2_path), link_path)
+ sub2_tree = (sub2_path, ["link"], ["tmp3"])
+ else:
+ sub2_tree = (sub2_path, [], ["tmp3"])
+
+ # Walk top-down.
+ all = list(os.walk(walk_path))
+ self.assertEqual(len(all), 4)
+ # We can't know which order SUB1 and SUB2 will appear in.
+ # Not flipped: TESTFN, SUB1, SUB11, SUB2
+ # flipped: TESTFN, SUB2, SUB1, SUB11
+ flipped = all[0][1][0] != "SUB1"
+ all[0][1].sort()
+ self.assertEqual(all[0], (walk_path, ["SUB1", "SUB2"], ["tmp1"]))
+ self.assertEqual(all[1 + flipped], (sub1_path, ["SUB11"], ["tmp2"]))
+ self.assertEqual(all[2 + flipped], (sub11_path, [], []))
+ self.assertEqual(all[3 - 2 * flipped], sub2_tree)
+
+ # Prune the search.
+ all = []
+ for root, dirs, files in os.walk(walk_path):
+ all.append((root, dirs, files))
+ # Don't descend into SUB1.
+ if 'SUB1' in dirs:
+ # Note that this also mutates the dirs we appended to all!
+ dirs.remove('SUB1')
+ self.assertEqual(len(all), 2)
+ self.assertEqual(all[0], (walk_path, ["SUB2"], ["tmp1"]))
+ self.assertEqual(all[1], sub2_tree)
+
+ # Walk bottom-up.
+ all = list(os.walk(walk_path, topdown=False))
+ self.assertEqual(len(all), 4)
+ # We can't know which order SUB1 and SUB2 will appear in.
+ # Not flipped: SUB11, SUB1, SUB2, TESTFN
+ # flipped: SUB2, SUB11, SUB1, TESTFN
+ flipped = all[3][1][0] != "SUB1"
+ all[3][1].sort()
+ self.assertEqual(all[3], (walk_path, ["SUB1", "SUB2"], ["tmp1"]))
+ self.assertEqual(all[flipped], (sub11_path, [], []))
+ self.assertEqual(all[flipped + 1], (sub1_path, ["SUB11"], ["tmp2"]))
+ self.assertEqual(all[2 - 2 * flipped], sub2_tree)
+
+ if hasattr(os, "symlink"):
+ # Walk, following symlinks.
+ for root, dirs, files in os.walk(walk_path, followlinks=True):
+ if root == link_path:
+ self.assertEqual(dirs, [])
+ self.assertEqual(files, ["tmp4"])
+ break
+ else:
+ self.fail("Didn't follow symlink with followlinks=True")
+
+ def tearDown(self):
+ # Tear everything down. This is a decent use for bottom-up on
+ # Windows, which doesn't have a recursive delete command. The
+ # (not so) subtlety is that rmdir will fail unless the dir's
+ # kids are removed first, so bottom up is essential.
+ for root, dirs, files in os.walk(test_support.TESTFN, topdown=False):
+ for name in files:
+ os.remove(os.path.join(root, name))
+ for name in dirs:
+ dirname = os.path.join(root, name)
+ if not os.path.islink(dirname):
+ os.rmdir(dirname)
+ else:
+ os.remove(dirname)
+ os.rmdir(test_support.TESTFN)
+
+class MakedirTests (unittest.TestCase):
+ def setUp(self):
+ os.mkdir(test_support.TESTFN)
+
+ def test_makedir(self):
+ base = test_support.TESTFN
+ path = os.path.join(base, 'dir1', 'dir2', 'dir3')
+ os.makedirs(path) # Should work
+ path = os.path.join(base, 'dir1', 'dir2', 'dir3', 'dir4')
+ os.makedirs(path)
+
+ # Try paths with a '.' in them
+ self.assertRaises(OSError, os.makedirs, os.curdir)
+ path = os.path.join(base, 'dir1', 'dir2', 'dir3', 'dir4', 'dir5', os.curdir)
+ os.makedirs(path)
+ path = os.path.join(base, 'dir1', os.curdir, 'dir2', 'dir3', 'dir4',
+ 'dir5', 'dir6')
+ os.makedirs(path)
+
+
+
+
+ def tearDown(self):
+ path = os.path.join(test_support.TESTFN, 'dir1', 'dir2', 'dir3',
+ 'dir4', 'dir5', 'dir6')
+ # If the tests failed, the bottom-most directory ('../dir6')
+ # may not have been created, so we look for the outermost directory
+ # that exists.
+ while not os.path.exists(path) and path != test_support.TESTFN:
+ path = os.path.dirname(path)
+
+ os.removedirs(path)
+
+class DevNullTests (unittest.TestCase):
+ def test_devnull(self):
+ f = file(os.devnull, 'w')
+ f.write('hello')
+ f.close()
+ f = file(os.devnull, 'r')
+ self.assertEqual(f.read(), '')
+ f.close()
+
+class URandomTests (unittest.TestCase):
+ def test_urandom(self):
+ try:
+ self.assertEqual(len(os.urandom(1)), 1)
+ self.assertEqual(len(os.urandom(10)), 10)
+ self.assertEqual(len(os.urandom(100)), 100)
+ self.assertEqual(len(os.urandom(1000)), 1000)
+ # see http://bugs.python.org/issue3708
+ self.assertRaises(TypeError, os.urandom, 0.9)
+ self.assertRaises(TypeError, os.urandom, 1.1)
+ self.assertRaises(TypeError, os.urandom, 2.0)
+ except NotImplementedError:
+ pass
+
+ def test_execvpe_with_bad_arglist(self):
+ self.assertRaises(ValueError, os.execvpe, 'notepad', [], None)
+
+class Win32ErrorTests(unittest.TestCase):
+ def test_rename(self):
+ self.assertRaises(WindowsError, os.rename, test_support.TESTFN, test_support.TESTFN+".bak")
+
+ def test_remove(self):
+ self.assertRaises(WindowsError, os.remove, test_support.TESTFN)
+
+ def test_chdir(self):
+ self.assertRaises(WindowsError, os.chdir, test_support.TESTFN)
+
+ def test_mkdir(self):
+ f = open(test_support.TESTFN, "w")
+ try:
+ self.assertRaises(WindowsError, os.mkdir, test_support.TESTFN)
+ finally:
+ f.close()
+ os.unlink(test_support.TESTFN)
+
+ def test_utime(self):
+ self.assertRaises(WindowsError, os.utime, test_support.TESTFN, None)
+
+ def test_chmod(self):
+ self.assertRaises(WindowsError, os.chmod, test_support.TESTFN, 0)
+
+class TestInvalidFD(unittest.TestCase):
+ singles = ["fchdir", "fdopen", "dup", "fdatasync", "fstat",
+ "fstatvfs", "fsync", "tcgetpgrp", "ttyname"]
+ #singles.append("close")
+ #We omit close because it doesn'r raise an exception on some platforms
+ def get_single(f):
+ def helper(self):
+ if hasattr(os, f):
+ self.check(getattr(os, f))
+ return helper
+ for f in singles:
+ locals()["test_"+f] = get_single(f)
+
+ def check(self, f, *args):
+ try:
+ f(test_support.make_bad_fd(), *args)
+ except OSError as e:
+ self.assertEqual(e.errno, errno.EBADF)
+ else:
+ self.fail("%r didn't raise a OSError with a bad file descriptor"
+ % f)
+
+ def test_isatty(self):
+ if hasattr(os, "isatty"):
+ self.assertEqual(os.isatty(test_support.make_bad_fd()), False)
+
+ def test_closerange(self):
+ if hasattr(os, "closerange"):
+ fd = test_support.make_bad_fd()
+ # Make sure none of the descriptors we are about to close are
+ # currently valid (issue 6542).
+ for i in range(10):
+ try: os.fstat(fd+i)
+ except OSError:
+ pass
+ else:
+ break
+ if i < 2:
+ raise unittest.SkipTest(
+ "Unable to acquire a range of invalid file descriptors")
+ self.assertEqual(os.closerange(fd, fd + i-1), None)
+
+ def test_dup2(self):
+ if hasattr(os, "dup2"):
+ self.check(os.dup2, 20)
+
+ def test_fchmod(self):
+ if hasattr(os, "fchmod"):
+ self.check(os.fchmod, 0)
+
+ def test_fchown(self):
+ if hasattr(os, "fchown"):
+ self.check(os.fchown, -1, -1)
+
+ def test_fpathconf(self):
+ if hasattr(os, "fpathconf"):
+ self.check(os.fpathconf, "PC_NAME_MAX")
+
+ def test_ftruncate(self):
+ if hasattr(os, "ftruncate"):
+ self.check(os.ftruncate, 0)
+
+ def test_lseek(self):
+ if hasattr(os, "lseek"):
+ self.check(os.lseek, 0, 0)
+
+ def test_read(self):
+ if hasattr(os, "read"):
+ self.check(os.read, 1)
+
+ def test_tcsetpgrpt(self):
+ if hasattr(os, "tcsetpgrp"):
+ self.check(os.tcsetpgrp, 0)
+
+ def test_write(self):
+ if hasattr(os, "write"):
+ self.check(os.write, " ")
+
+if sys.platform != 'win32':
+ class Win32ErrorTests(unittest.TestCase):
+ pass
+
+ class PosixUidGidTests(unittest.TestCase):
+ if hasattr(os, 'setuid'):
+ def test_setuid(self):
+ if os.getuid() != 0:
+ self.assertRaises(os.error, os.setuid, 0)
+ self.assertRaises(OverflowError, os.setuid, 1<<32)
+
+ if hasattr(os, 'setgid'):
+ def test_setgid(self):
+ if os.getuid() != 0:
+ self.assertRaises(os.error, os.setgid, 0)
+ self.assertRaises(OverflowError, os.setgid, 1<<32)
+
+ if hasattr(os, 'seteuid'):
+ def test_seteuid(self):
+ if os.getuid() != 0:
+ self.assertRaises(os.error, os.seteuid, 0)
+ self.assertRaises(OverflowError, os.seteuid, 1<<32)
+
+ if hasattr(os, 'setegid'):
+ def test_setegid(self):
+ if os.getuid() != 0:
+ self.assertRaises(os.error, os.setegid, 0)
+ self.assertRaises(OverflowError, os.setegid, 1<<32)
+
+ if hasattr(os, 'setreuid'):
+ def test_setreuid(self):
+ if os.getuid() != 0:
+ self.assertRaises(os.error, os.setreuid, 0, 0)
+ self.assertRaises(OverflowError, os.setreuid, 1<<32, 0)
+ self.assertRaises(OverflowError, os.setreuid, 0, 1<<32)
+
+ def test_setreuid_neg1(self):
+ # Needs to accept -1. We run this in a subprocess to avoid
+ # altering the test runner's process state (issue8045).
+ subprocess.check_call([
+ sys.executable, '-c',
+ 'import os,sys;os.setreuid(-1,-1);sys.exit(0)'])
+
+ if hasattr(os, 'setregid'):
+ def test_setregid(self):
+ if os.getuid() != 0:
+ self.assertRaises(os.error, os.setregid, 0, 0)
+ self.assertRaises(OverflowError, os.setregid, 1<<32, 0)
+ self.assertRaises(OverflowError, os.setregid, 0, 1<<32)
+
+ def test_setregid_neg1(self):
+ # Needs to accept -1. We run this in a subprocess to avoid
+ # altering the test runner's process state (issue8045).
+ subprocess.check_call([
+ sys.executable, '-c',
+ 'import os,sys;os.setregid(-1,-1);sys.exit(0)'])
+else:
+ class PosixUidGidTests(unittest.TestCase):
+ pass
+
+ at unittest.skipUnless(sys.platform == "win32", "Win32 specific tests")
+class Win32KillTests(unittest.TestCase):
+ def _kill(self, sig):
+ # Start sys.executable as a subprocess and communicate from the
+ # subprocess to the parent that the interpreter is ready. When it
+ # becomes ready, send *sig* via os.kill to the subprocess and check
+ # that the return code is equal to *sig*.
+ import ctypes
+ from ctypes import wintypes
+ import msvcrt
+
+ # Since we can't access the contents of the process' stdout until the
+ # process has exited, use PeekNamedPipe to see what's inside stdout
+ # without waiting. This is done so we can tell that the interpreter
+ # is started and running at a point where it could handle a signal.
+ PeekNamedPipe = ctypes.windll.kernel32.PeekNamedPipe
+ PeekNamedPipe.restype = wintypes.BOOL
+ PeekNamedPipe.argtypes = (wintypes.HANDLE, # Pipe handle
+ ctypes.POINTER(ctypes.c_char), # stdout buf
+ wintypes.DWORD, # Buffer size
+ ctypes.POINTER(wintypes.DWORD), # bytes read
+ ctypes.POINTER(wintypes.DWORD), # bytes avail
+ ctypes.POINTER(wintypes.DWORD)) # bytes left
+ msg = "running"
+ proc = subprocess.Popen([sys.executable, "-c",
+ "import sys;"
+ "sys.stdout.write('{}');"
+ "sys.stdout.flush();"
+ "input()".format(msg)],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ stdin=subprocess.PIPE)
+ self.addCleanup(proc.stdout.close)
+ self.addCleanup(proc.stderr.close)
+ self.addCleanup(proc.stdin.close)
+
+ count, max = 0, 100
+ while count < max and proc.poll() is None:
+ # Create a string buffer to store the result of stdout from the pipe
+ buf = ctypes.create_string_buffer(len(msg))
+ # Obtain the text currently in proc.stdout
+ # Bytes read/avail/left are left as NULL and unused
+ rslt = PeekNamedPipe(msvcrt.get_osfhandle(proc.stdout.fileno()),
+ buf, ctypes.sizeof(buf), None, None, None)
+ self.assertNotEqual(rslt, 0, "PeekNamedPipe failed")
+ if buf.value:
+ self.assertEqual(msg, buf.value)
+ break
+ time.sleep(0.1)
+ count += 1
+ else:
+ self.fail("Did not receive communication from the subprocess")
+
+ os.kill(proc.pid, sig)
+ self.assertEqual(proc.wait(), sig)
+
+ def test_kill_sigterm(self):
+ # SIGTERM doesn't mean anything special, but make sure it works
+ self._kill(signal.SIGTERM)
+
+ def test_kill_int(self):
+ # os.kill on Windows can take an int which gets set as the exit code
+ self._kill(100)
+
+ def _kill_with_event(self, event, name):
+ tagname = "test_os_%s" % uuid.uuid1()
+ m = mmap.mmap(-1, 1, tagname)
+ m[0] = '0'
+ # Run a script which has console control handling enabled.
+ proc = subprocess.Popen([sys.executable,
+ os.path.join(os.path.dirname(__file__),
+ "win_console_handler.py"), tagname],
+ creationflags=subprocess.CREATE_NEW_PROCESS_GROUP)
+ # Let the interpreter startup before we send signals. See #3137.
+ count, max = 0, 20
+ while count < max and proc.poll() is None:
+ if m[0] == '1':
+ break
+ time.sleep(0.5)
+ count += 1
+ else:
+ self.fail("Subprocess didn't finish initialization")
+ os.kill(proc.pid, event)
+ # proc.send_signal(event) could also be done here.
+ # Allow time for the signal to be passed and the process to exit.
+ time.sleep(0.5)
+ if not proc.poll():
+ # Forcefully kill the process if we weren't able to signal it.
+ os.kill(proc.pid, signal.SIGINT)
+ self.fail("subprocess did not stop on {}".format(name))
+
+ @unittest.skip("subprocesses aren't inheriting CTRL+C property")
+ def test_CTRL_C_EVENT(self):
+ from ctypes import wintypes
+ import ctypes
+
+ # Make a NULL value by creating a pointer with no argument.
+ NULL = ctypes.POINTER(ctypes.c_int)()
+ SetConsoleCtrlHandler = ctypes.windll.kernel32.SetConsoleCtrlHandler
+ SetConsoleCtrlHandler.argtypes = (ctypes.POINTER(ctypes.c_int),
+ wintypes.BOOL)
+ SetConsoleCtrlHandler.restype = wintypes.BOOL
+
+ # Calling this with NULL and FALSE causes the calling process to
+ # handle CTRL+C, rather than ignore it. This property is inherited
+ # by subprocesses.
+ SetConsoleCtrlHandler(NULL, 0)
+
+ self._kill_with_event(signal.CTRL_C_EVENT, "CTRL_C_EVENT")
+
+ def test_CTRL_BREAK_EVENT(self):
+ self._kill_with_event(signal.CTRL_BREAK_EVENT, "CTRL_BREAK_EVENT")
+
+
+def test_main():
+ test_support.run_unittest(
+ FileTests,
+ TemporaryFileTests,
+ StatAttributeTests,
+ EnvironTests,
+ WalkTests,
+ MakedirTests,
+ DevNullTests,
+ URandomTests,
+ Win32ErrorTests,
+ TestInvalidFD,
+ PosixUidGidTests,
+ Win32KillTests
+ )
+
+if __name__ == "__main__":
+ test_main()
diff --git a/lib-python/modified-2.7/tarfile.py b/lib-python/modified-2.7/tarfile.py
--- a/lib-python/modified-2.7/tarfile.py
+++ b/lib-python/modified-2.7/tarfile.py
@@ -425,10 +425,16 @@
raise CompressionError("zlib module is not available")
self.zlib = zlib
self.crc = zlib.crc32("") & 0xffffffffL
- if mode == "r":
- self._init_read_gz()
- else:
- self._init_write_gz()
+ try:
+ if mode == "r":
+ self._init_read_gz()
+ else:
+ self._init_write_gz()
+ except:
+ if not self._extfileobj:
+ fileobj.close()
+ self.closed = True
+ raise
if comptype == "bz2":
try:
@@ -1682,13 +1688,14 @@
if filemode not in "rw":
raise ValueError("mode must be 'r' or 'w'")
-
- t = cls(name, filemode,
- _Stream(name, filemode, comptype, fileobj, bufsize),
- **kwargs)
- t._extfileobj = False
- return t
-
+ fid = _Stream(name, filemode, comptype, fileobj, bufsize)
+ try:
+ t = cls(name, filemode, fid, **kwargs)
+ t._extfileobj = False
+ return t
+ except:
+ fid.close()
+ raise
elif mode in "aw":
return cls.taropen(name, mode, fileobj, **kwargs)
@@ -1715,13 +1722,18 @@
gzip.GzipFile
except (ImportError, AttributeError):
raise CompressionError("gzip module is not available")
-
+ gz_fid = None
try:
- t = cls.taropen(name, mode,
- gzip.GzipFile(name, mode, compresslevel, fileobj),
- **kwargs)
+ gz_fid = gzip.GzipFile(name, mode, compresslevel, fileobj)
+ t = cls.taropen(name, mode, gz_fid, **kwargs)
except IOError:
+ if gz_fid:
+ gz_fid.close()
raise ReadError("not a gzip file")
+ except:
+ if gz_fid:
+ gz_fid.close()
+ raise
t._extfileobj = False
return t
@@ -1738,15 +1750,21 @@
except ImportError:
raise CompressionError("bz2 module is not available")
- if fileobj is not None:
- fileobj = _BZ2Proxy(fileobj, mode)
- else:
- fileobj = bz2.BZ2File(name, mode, compresslevel=compresslevel)
+ try:
+ if fileobj is not None:
+ bzfileobj = _BZ2Proxy(fileobj, mode)
+ else:
+ bzfileobj = bz2.BZ2File(name, mode, compresslevel=compresslevel)
+ t = cls.taropen(name, mode, bzfileobj, **kwargs)
- try:
- t = cls.taropen(name, mode, fileobj, **kwargs)
except (IOError, EOFError):
+ if fileobj is None:
+ bzfileobj.close()
raise ReadError("not a bzip2 file")
+ except:
+ if fileobj is None:
+ bzfileobj.close()
+ raise
t._extfileobj = False
return t
diff --git a/lib-python/modified-2.7/test/test_tarfile.py b/lib-python/modified-2.7/test/test_tarfile.py
--- a/lib-python/modified-2.7/test/test_tarfile.py
+++ b/lib-python/modified-2.7/test/test_tarfile.py
@@ -340,7 +340,8 @@
# constructor in case of an error. For the test we rely on
# the fact that opening an empty file raises a ReadError.
empty = os.path.join(TEMPDIR, "empty")
- open(empty, "wb").write("")
+ with open(empty, "wb") as fid:
+ fid.write("")
try:
tar = object.__new__(tarfile.TarFile)
diff --git a/lib-python/2.7/test/test_zipfile.py b/lib-python/modified-2.7/test/test_zipfile.py
copy from lib-python/2.7/test/test_zipfile.py
copy to lib-python/modified-2.7/test/test_zipfile.py
--- a/lib-python/2.7/test/test_zipfile.py
+++ b/lib-python/modified-2.7/test/test_zipfile.py
@@ -234,8 +234,9 @@
# Read the ZIP archive
with zipfile.ZipFile(f, "r") as zipfp:
- for line, zipline in zip(self.line_gen, zipfp.open(TESTFN)):
- self.assertEqual(zipline, line + '\n')
+ with zipfp.open(TESTFN) as f:
+ for line, zipline in zip(self.line_gen, f):
+ self.assertEqual(zipline, line + '\n')
def test_readline_read_stored(self):
# Issue #7610: calls to readline() interleaved with calls to read().
@@ -340,7 +341,8 @@
produces the expected result."""
with zipfile.ZipFile(TESTFN2, "w") as zipfp:
zipfp.write(TESTFN)
- self.assertEqual(zipfp.read(TESTFN), open(TESTFN).read())
+ with open(TESTFN) as f:
+ self.assertEqual(zipfp.read(TESTFN), f.read())
@skipUnless(zlib, "requires zlib")
def test_per_file_compression(self):
@@ -382,7 +384,8 @@
self.assertEqual(writtenfile, correctfile)
# make sure correct data is in correct file
- self.assertEqual(fdata, open(writtenfile, "rb").read())
+ with open(writtenfile, "rb") as fid:
+ self.assertEqual(fdata, fid.read())
os.remove(writtenfile)
# remove the test file subdirectories
@@ -401,24 +404,25 @@
else:
outfile = os.path.join(os.getcwd(), fpath)
- self.assertEqual(fdata, open(outfile, "rb").read())
+ with open(outfile, "rb") as fid:
+ self.assertEqual(fdata, fid.read())
os.remove(outfile)
# remove the test file subdirectories
shutil.rmtree(os.path.join(os.getcwd(), 'ziptest2dir'))
def test_writestr_compression(self):
- zipfp = zipfile.ZipFile(TESTFN2, "w")
- zipfp.writestr("a.txt", "hello world", compress_type=zipfile.ZIP_STORED)
- if zlib:
- zipfp.writestr("b.txt", "hello world", compress_type=zipfile.ZIP_DEFLATED)
+ with zipfile.ZipFile(TESTFN2, "w") as zipfp:
+ zipfp.writestr("a.txt", "hello world", compress_type=zipfile.ZIP_STORED)
+ if zlib:
+ zipfp.writestr("b.txt", "hello world", compress_type=zipfile.ZIP_DEFLATED)
- info = zipfp.getinfo('a.txt')
- self.assertEqual(info.compress_type, zipfile.ZIP_STORED)
+ info = zipfp.getinfo('a.txt')
+ self.assertEqual(info.compress_type, zipfile.ZIP_STORED)
- if zlib:
- info = zipfp.getinfo('b.txt')
- self.assertEqual(info.compress_type, zipfile.ZIP_DEFLATED)
+ if zlib:
+ info = zipfp.getinfo('b.txt')
+ self.assertEqual(info.compress_type, zipfile.ZIP_DEFLATED)
def zip_test_writestr_permissions(self, f, compression):
@@ -646,7 +650,8 @@
def test_write_non_pyfile(self):
with zipfile.PyZipFile(TemporaryFile(), "w") as zipfp:
- open(TESTFN, 'w').write('most definitely not a python file')
+ with open(TESTFN, 'w') as f:
+ f.write('most definitely not a python file')
self.assertRaises(RuntimeError, zipfp.writepy, TESTFN)
os.remove(TESTFN)
@@ -795,7 +800,8 @@
self.assertRaises(RuntimeError, zipf.open, "foo.txt")
self.assertRaises(RuntimeError, zipf.testzip)
self.assertRaises(RuntimeError, zipf.writestr, "bogus.txt", "bogus")
- open(TESTFN, 'w').write('zipfile test data')
+ with open(TESTFN, 'w') as fp:
+ fp.write('zipfile test data')
self.assertRaises(RuntimeError, zipf.write, TESTFN)
def test_bad_constructor_mode(self):
@@ -803,7 +809,6 @@
self.assertRaises(RuntimeError, zipfile.ZipFile, TESTFN, "q")
def test_bad_open_mode(self):
- """Check that bad modes passed to ZipFile.open are caught."""
with zipfile.ZipFile(TESTFN, mode="w") as zipf:
zipf.writestr("foo.txt", "O, for a Muse of Fire!")
@@ -851,7 +856,6 @@
def test_comments(self):
"""Check that comments on the archive are handled properly."""
-
# check default comment is empty
with zipfile.ZipFile(TESTFN, mode="w") as zipf:
self.assertEqual(zipf.comment, '')
@@ -953,14 +957,16 @@
with zipfile.ZipFile(TESTFN, mode="w") as zipf:
pass
try:
- zipf = zipfile.ZipFile(TESTFN, mode="r")
+ with zipfile.ZipFile(TESTFN, mode="r") as zipf:
+ pass
except zipfile.BadZipfile:
self.fail("Unable to create empty ZIP file in 'w' mode")
with zipfile.ZipFile(TESTFN, mode="a") as zipf:
pass
try:
- zipf = zipfile.ZipFile(TESTFN, mode="r")
+ with zipfile.ZipFile(TESTFN, mode="r") as zipf:
+ pass
except:
self.fail("Unable to create empty ZIP file in 'a' mode")
@@ -1160,6 +1166,8 @@
data1 += zopen1.read(500)
data2 += zopen2.read(500)
self.assertEqual(data1, data2)
+ zopen1.close()
+ zopen2.close()
def test_different_file(self):
# Verify that (when the ZipFile is in control of creating file objects)
@@ -1207,9 +1215,9 @@
def test_store_dir(self):
os.mkdir(os.path.join(TESTFN2, "x"))
- zipf = zipfile.ZipFile(TESTFN, "w")
- zipf.write(os.path.join(TESTFN2, "x"), "x")
- self.assertTrue(zipf.filelist[0].filename.endswith("x/"))
+ with zipfile.ZipFile(TESTFN, "w") as zipf:
+ zipf.write(os.path.join(TESTFN2, "x"), "x")
+ self.assertTrue(zipf.filelist[0].filename.endswith("x/"))
def tearDown(self):
shutil.rmtree(TESTFN2)
@@ -1226,7 +1234,8 @@
for n, s in enumerate(self.seps):
self.arcdata[s] = s.join(self.line_gen) + s
self.arcfiles[s] = '%s-%d' % (TESTFN, n)
- open(self.arcfiles[s], "wb").write(self.arcdata[s])
+ with open(self.arcfiles[s], "wb") as f:
+ f.write(self.arcdata[s])
def make_test_archive(self, f, compression):
# Create the ZIP archive
@@ -1295,8 +1304,9 @@
# Read the ZIP archive
with zipfile.ZipFile(f, "r") as zipfp:
for sep, fn in self.arcfiles.items():
- for line, zipline in zip(self.line_gen, zipfp.open(fn, "rU")):
- self.assertEqual(zipline, line + '\n')
+ with zipfp.open(fn, "rU") as f:
+ for line, zipline in zip(self.line_gen, f):
+ self.assertEqual(zipline, line + '\n')
def test_read_stored(self):
for f in (TESTFN2, TemporaryFile(), StringIO()):
diff --git a/lib-python/2.7/zipfile.py b/lib-python/modified-2.7/zipfile.py
copy from lib-python/2.7/zipfile.py
copy to lib-python/modified-2.7/zipfile.py
--- a/lib-python/2.7/zipfile.py
+++ b/lib-python/modified-2.7/zipfile.py
@@ -648,6 +648,10 @@
return data
+class ZipExtFileWithClose(ZipExtFile):
+ def close(self):
+ self._fileobj.close()
+
class ZipFile:
""" Class with methods to open, read, write, close, list zip files.
@@ -843,9 +847,9 @@
try:
# Read by chunks, to avoid an OverflowError or a
# MemoryError with very large embedded files.
- f = self.open(zinfo.filename, "r")
- while f.read(chunk_size): # Check CRC-32
- pass
+ with self.open(zinfo.filename, "r") as f:
+ while f.read(chunk_size): # Check CRC-32
+ pass
except BadZipfile:
return zinfo.filename
@@ -864,7 +868,9 @@
def read(self, name, pwd=None):
"""Return file bytes (as a string) for name."""
- return self.open(name, "r", pwd).read()
+ with self.open(name, "r", pwd) as f:
+ retval = f.read()
+ return retval
def open(self, name, mode="r", pwd=None):
"""Return file-like object for 'name'."""
@@ -881,59 +887,66 @@
else:
zef_file = open(self.filename, 'rb')
- # Make sure we have an info object
- if isinstance(name, ZipInfo):
- # 'name' is already an info object
- zinfo = name
+ try:
+ # Make sure we have an info object
+ if isinstance(name, ZipInfo):
+ # 'name' is already an info object
+ zinfo = name
+ else:
+ # Get info object for name
+ zinfo = self.getinfo(name)
+
+ zef_file.seek(zinfo.header_offset, 0)
+
+ # Skip the file header:
+ fheader = zef_file.read(sizeFileHeader)
+ if fheader[0:4] != stringFileHeader:
+ raise BadZipfile, "Bad magic number for file header"
+
+ fheader = struct.unpack(structFileHeader, fheader)
+ fname = zef_file.read(fheader[_FH_FILENAME_LENGTH])
+ if fheader[_FH_EXTRA_FIELD_LENGTH]:
+ zef_file.read(fheader[_FH_EXTRA_FIELD_LENGTH])
+
+ if fname != zinfo.orig_filename:
+ raise BadZipfile, \
+ 'File name in directory "%s" and header "%s" differ.' % (
+ zinfo.orig_filename, fname)
+
+ # check for encrypted flag & handle password
+ is_encrypted = zinfo.flag_bits & 0x1
+ zd = None
+ if is_encrypted:
+ if not pwd:
+ pwd = self.pwd
+ if not pwd:
+ raise RuntimeError, "File %s is encrypted, " \
+ "password required for extraction" % name
+
+ zd = _ZipDecrypter(pwd)
+ # The first 12 bytes in the cypher stream is an encryption header
+ # used to strengthen the algorithm. The first 11 bytes are
+ # completely random, while the 12th contains the MSB of the CRC,
+ # or the MSB of the file time depending on the header type
+ # and is used to check the correctness of the password.
+ bytes = zef_file.read(12)
+ h = map(zd, bytes[0:12])
+ if zinfo.flag_bits & 0x8:
+ # compare against the file type from extended local headers
+ check_byte = (zinfo._raw_time >> 8) & 0xff
+ else:
+ # compare against the CRC otherwise
+ check_byte = (zinfo.CRC >> 24) & 0xff
+ if ord(h[11]) != check_byte:
+ raise RuntimeError("Bad password for file", name)
+ except:
+ if not self._filePassed:
+ zef_file.close()
+ raise
+ if self._filePassed:
+ return ZipExtFile(zef_file, mode, zinfo, zd)
else:
- # Get info object for name
- zinfo = self.getinfo(name)
-
- zef_file.seek(zinfo.header_offset, 0)
-
- # Skip the file header:
- fheader = zef_file.read(sizeFileHeader)
- if fheader[0:4] != stringFileHeader:
- raise BadZipfile, "Bad magic number for file header"
-
- fheader = struct.unpack(structFileHeader, fheader)
- fname = zef_file.read(fheader[_FH_FILENAME_LENGTH])
- if fheader[_FH_EXTRA_FIELD_LENGTH]:
- zef_file.read(fheader[_FH_EXTRA_FIELD_LENGTH])
-
- if fname != zinfo.orig_filename:
- raise BadZipfile, \
- 'File name in directory "%s" and header "%s" differ.' % (
- zinfo.orig_filename, fname)
-
- # check for encrypted flag & handle password
- is_encrypted = zinfo.flag_bits & 0x1
- zd = None
- if is_encrypted:
- if not pwd:
- pwd = self.pwd
- if not pwd:
- raise RuntimeError, "File %s is encrypted, " \
- "password required for extraction" % name
-
- zd = _ZipDecrypter(pwd)
- # The first 12 bytes in the cypher stream is an encryption header
- # used to strengthen the algorithm. The first 11 bytes are
- # completely random, while the 12th contains the MSB of the CRC,
- # or the MSB of the file time depending on the header type
- # and is used to check the correctness of the password.
- bytes = zef_file.read(12)
- h = map(zd, bytes[0:12])
- if zinfo.flag_bits & 0x8:
- # compare against the file type from extended local headers
- check_byte = (zinfo._raw_time >> 8) & 0xff
- else:
- # compare against the CRC otherwise
- check_byte = (zinfo.CRC >> 24) & 0xff
- if ord(h[11]) != check_byte:
- raise RuntimeError("Bad password for file", name)
-
- return ZipExtFile(zef_file, mode, zinfo, zd)
+ return ZipExtFileWithClose(zef_file, mode, zinfo, zd)
def extract(self, member, path=None, pwd=None):
"""Extract a member from the archive to the current working directory,
@@ -989,7 +1002,6 @@
if not os.path.isdir(targetpath):
os.mkdir(targetpath)
return targetpath
-
source = self.open(member, pwd=pwd)
target = file(targetpath, "wb")
shutil.copyfileobj(source, target)
diff --git a/lib_pypy/_ctypes/builtin.py b/lib_pypy/_ctypes/builtin.py
--- a/lib_pypy/_ctypes/builtin.py
+++ b/lib_pypy/_ctypes/builtin.py
@@ -3,7 +3,8 @@
try:
from thread import _local as local
except ImportError:
- local = object # no threads
+ class local(object): # no threads
+ pass
class ConvMode:
encoding = 'ascii'
diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/cppyy.rst
@@ -0,0 +1,333 @@
+============================
+cppyy: C++ bindings for PyPy
+============================
+
+The cppyy module provides C++ bindings for PyPy by using the reflection
+information extracted from C++ header files by means of the
+`Reflex package`_.
+For this to work, you have to both install Reflex and build PyPy from the
+reflex-support branch.
+As indicated by this being a branch, support for Reflex is still
+experimental.
+However, it is functional enough to put it in the hands of those who want
+to give it a try.
+In the medium term, cppyy will move away from Reflex and instead use
+`cling`_ as its backend, which is based on `llvm`_.
+Although that will change the logistics on the generation of reflection
+information, it will not change the python-side interface.
+
+.. _`Reflex package`: http://root.cern.ch/drupal/content/reflex
+.. _`cling`: http://root.cern.ch/drupal/content/cling
+.. _`llvm`: http://llvm.org/
+
+
+Installation
+============
+
+For now, the easiest way of getting the latest version of Reflex, is by
+installing the ROOT package.
+Besides getting the latest version of Reflex, another advantage is that with
+the full ROOT package, you can also use your Reflex-bound code on `CPython`_.
+`Download`_ a binary or install from `source`_.
+Some Linux and Mac systems may have ROOT provided in the list of scientific
+software of their packager.
+A current, standalone version of Reflex should be provided at some point,
+once the dependencies and general packaging have been thought out.
+Also, make sure you have a version of `gccxml`_ installed, which is most
+easily provided by the packager of your system.
+If you read up on gccxml, you'll probably notice that it is no longer being
+developed and hence will not provide C++11 support.
+That's why the medium term plan is to move to `cling`_.
+
+.. _`Download`: http://root.cern.ch/drupal/content/downloading-root
+.. _`source`: http://root.cern.ch/drupal/content/installing-root-source
+.. _`gccxml`: http://www.gccxml.org
+
+Next, get the `PyPy sources`_, select the reflex-support branch, and build
+pypy-c.
+For the build to succeed, the ``$ROOTSYS`` environment variable must point to
+the location of your ROOT installation::
+
+ $ hg clone https://bitbucket.org/pypy/pypy
+ $ cd pypy
+ $ hg up reflex-support
+ $ cd pypy/translator/goal
+ $ python translate.py -O jit --gcrootfinder=shadowstack targetpypystandalone.py --withmod-cppyy
+
+This will build a ``pypy-c`` that includes the cppyy module, and through that,
+Reflex support.
+Of course, if you already have a pre-built version of the ``pypy`` interpreter,
+you can use that for the translation rather than ``python``.
+
+.. _`PyPy sources`: https://bitbucket.org/pypy/pypy/overview
+
+
+Basic example
+=============
+
+Now test with a trivial example whether all packages are properly installed
+and functional.
+First, create a C++ header file with some class in it (note that all functions
+are made inline for convenience; a real-world example would of course have a
+corresponding source file)::
+
+ $ cat MyClass.h
+ class MyClass {
+ public:
+ MyClass(int i = -99) : m_myint(i) {}
+
+ int GetMyInt() { return m_myint; }
+ void SetMyInt(int i) { m_myint = i; }
+
+ public:
+ int m_myint;
+ };
+
+Then, generate the bindings using ``genreflex`` (part of ROOT), and compile the
+code::
+
+ $ genreflex MyClass.h
+ $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyClass_rflx.cpp -o libMyClassDict.so
+
+Now you're ready to use the bindings.
+Since the bindings are designed to look pythonistic, it should be
+straightforward::
+
+ $ pypy-c
+ >>>> import cppyy
+ >>>> cppyy.load_reflection_info("libMyClassDict.so")
+ <CPPLibrary object at 0xb6fd7c4c>
+ >>>> myinst = cppyy.gbl.MyClass(42)
+ >>>> print myinst.GetMyInt()
+ 42
+ >>>> myinst.SetMyInt(33)
+ >>>> print myinst.m_myint
+ 33
+ >>>> myinst.m_myint = 77
+ >>>> print myinst.GetMyInt()
+ 77
+ >>>> help(cppyy.gbl.MyClass) # shows that normal python introspection works
+
+That's all there is to it!
+
+
+Features
+========
+
+The following is not meant to be an exhaustive list, since cppyy is still
+under active development.
+Furthermore, the intention is that every feature is as natural as possible on
+the python side, so if you find something missing in the list below, simply
+try it out.
+It is not always possible to provide exact mapping between python and C++
+(active memory management is one such case), but by and large, if the use of a
+feature does not strike you as obvious, it is more likely to simply be a bug.
+That is a strong statement to make, but also a worthy goal.
+
+* **abstract classes**: Are represented as python classes, since they are
+ needed to complete the inheritance hierarchies, but will raise an exception
+ if an attempt is made to instantiate from them.
+
+* **arrays**: Supported for builtin data types only, as used from module
+ ``array``.
+ Out-of-bounds checking is limited to those cases where the size is known at
+ compile time (and hence part of the reflection info).
+
+* **builtin data types**: Map onto the expected equivalent python types, with
+ the caveat that there may be size differences, and thus it is possible that
+ exceptions are raised if an overflow is detected.
+
+* **casting**: Is supposed to be unnecessary.
+ Object pointer returns from functions provide the most derived class known
+ in the hierarchy of the object being returned.
+ This is important to preserve object identity as well as to make casting,
+ a pure C++ feature after all, superfluous.
+
+* **classes and structs**: Get mapped onto python classes, where they can be
+ instantiated as expected.
+ If classes are inner classes or live in a namespace, their naming and
+ location will reflect that.
+
+* **data members**: Public data members are represented as python properties
+ and provide read and write access on instances as expected.
+
+* **default arguments**: C++ default arguments work as expected, but python
+ keywords are not supported.
+ It is technically possible to support keywords, but for the C++ interface,
+ the formal argument names have no meaning and are not considered part of the
+ API, hence it is not a good idea to use keywords.
+
+* **doc strings**: The doc string of a method or function contains the C++
+ arguments and return types of all overloads of that name, as applicable.
+
+* **functions**: Work as expected and live in their appropriate namespace
+ (which can be the global one, ``cppyy.gbl``).
+
+* **inheritance**: All combinations of inheritance on the C++ (single,
+ multiple, virtual) are supported in the binding.
+ However, new python classes can only use single inheritance from a bound C++
+ class.
+ Multiple inheritance would introduce two "this" pointers in the binding.
+ This is a current, not a fundamental, limitation.
+ The C++ side will not see any overridden methods on the python side, as
+ cross-inheritance is planned but not yet supported.
+
+* **methods**: Are represented as python methods and work as expected.
+ They are first class objects and can be bound to an instance.
+ Virtual C++ methods work as expected.
+ To select a specific virtual method, do like with normal python classes
+ that override methods: select it from the class that you need, rather than
+ calling the method on the instance.
+
+* **namespaces**: Are represented as python classes.
+ Namespaces are more open-ended than classes, so sometimes initial access may
+ result in updates as data and functions are looked up and constructed
+ lazily.
+ Thus the result of ``dir()`` on a namespace should not be relied upon: it
+ only shows the already accessed members. (TODO: to be fixed by implementing
+ __dir__.)
+ The global namespace is ``cppyy.gbl``.
+
+* **operator conversions**: If defined in the C++ class and a python
+ equivalent exists (i.e. all builtin integer and floating point types, as well
+ as ``bool``), it will map onto that python conversion.
+ Note that ``char*`` is mapped onto ``__str__``.
+
+* **operator overloads**: If defined in the C++ class and if a python
+ equivalent is available (not always the case, think e.g. of ``operator||``),
+ then they work as expected.
+ Special care needs to be taken for global operator overloads in C++: first,
+ make sure that they are actually reflected, especially for the global
+ overloads for ``operator==`` and ``operator!=`` of STL iterators in the case
+ of gcc.
+ Second, make sure that reflection info is loaded in the proper order.
+ I.e. that these global overloads are available before use.
+
+* **pointers**: For builtin data types, see arrays.
+ For objects, a pointer to an object and an object looks the same, unless
+ the pointer is a data member.
+ In that case, assigning to the data member will cause a copy of the pointer
+ and care should be taken about the object's life time.
+ If a pointer is a global variable, the C++ side can replace the underlying
+ object and the python side will immediately reflect that.
+
+* **static data members**: Are represented as python property objects on the
+ class and the meta-class.
+ Both read and write access is as expected.
+
+* **static methods**: Are represented as python's ``staticmethod`` objects
+ and can be called both from the class as well as from instances.
+
+* **strings**: The std::string class is considered a builtin C++ type and
+ mixes quite well with python's str.
+ Python's str can be passed where a ``const char*`` is expected, and an str
+ will be returned if the return type is ``const char*``.
+
+* **templated classes**: Are represented in a meta-class style in python.
+ This looks a little bit confusing, but conceptually is rather natural.
+ For example, given the class ``std::vector<int>``, the meta-class part would
+ be ``std.vector`` in python.
+ Then, to get the instantiation on ``int``, do ``std.vector(int)`` and to
+ create an instance of that class, do ``std.vector(int)()``.
+ Note that templates can be build up by handing actual types to the class
+ instantiation (as done in this vector example), or by passing in the list of
+ template arguments as a string.
+ The former is a lot easier to work with if you have template instantiations
+ using classes that themselves are templates (etc.) in the arguments.
+ All classes must already exist in the loaded reflection info.
+
+* **unary operators**: Are supported if a python equivalent exists, and if the
+ operator is defined in the C++ class.
+
+You can always find more detailed examples and see the full of supported
+features by looking at the tests in pypy/module/cppyy/test.
+
+If a feature or reflection info is missing, this is supposed to be handled
+gracefully.
+In fact, there are unit tests explicitly for this purpose (even as their use
+becomes less interesting over time, as the number of missing features
+decreases).
+Only when a missing feature is used, should there be an exception.
+For example, if no reflection info is available for a return type, then a
+class that has a method with that return type can still be used.
+Only that one specific method can not be used.
+
+
+The fast lane
+=============
+
+The following is an experimental feature of cppyy, and that makes it doubly
+experimental, so caveat emptor.
+With a slight modification of Reflex, it can provide function pointers for
+C++ methods, and hence allow PyPy to call those pointers directly, rather than
+calling C++ through a Reflex stub.
+This results in a rather significant speed-up.
+Mind you, the normal stub path is not exactly slow, so for now only use this
+out of curiosity or if you really need it.
+
+To install this patch of Reflex, locate the file genreflex-methptrgetter.patch
+in pypy/module/cppyy and apply it to the genreflex python scripts found in
+``$ROOTSYS/lib``::
+
+ $ cd $ROOTSYS/lib
+ $ patch -p2 < genreflex-methptrgetter.patch
+
+With this patch, ``genreflex`` will have grown the ``--with-methptrgetter``
+option.
+Use this option when running ``genreflex``, and add the
+``-Wno-pmf-conversions`` option to ``g++`` when compiling.
+The rest works the same way: the fast path will be used transparently (which
+also means that you can't actually find out whether it is in use, other than
+by running a micro-benchmark).
+
+
+CPython
+=======
+
+Most of the ideas in cppyy come originally from the `PyROOT`_ project.
+Although PyROOT does not support Reflex directly, it has an alter ego called
+"PyCintex" that, in a somewhat roundabout way, does.
+If you installed ROOT, rather than just Reflex, PyCintex should be available
+immediately if you add ``$ROOTSYS/lib`` to the ``PYTHONPATH`` environment
+variable.
+
+.. _`PyROOT`: http://root.cern.ch/drupal/content/pyroot
+
+There are a couple of minor differences between PyCintex and cppyy, most to do
+with naming.
+The one that you will run into directly, is that PyCintex uses a function
+called ``loadDictionary`` rather than ``load_reflection_info``.
+The reason for this is that Reflex calls the shared libraries that contain
+reflection info "dictionaries."
+However, in python, the name `dictionary` already has a well-defined meaning,
+so a more descriptive name was chosen for cppyy.
+In addition, PyCintex requires that the names of shared libraries so loaded
+start with "lib" in their name.
+The basic example above, rewritten for PyCintex thus goes like this::
+
+ $ python
+ >>> import PyCintex
+ >>> PyCintex.loadDictionary("libMyClassDict.so")
+ >>> myinst = PyCintex.gbl.MyClass(42)
+ >>> print myinst.GetMyInt()
+ 42
+ >>> myinst.SetMyInt(33)
+ >>> print myinst.m_myint
+ 33
+ >>> myinst.m_myint = 77
+ >>> print myinst.GetMyInt()
+ 77
+ >>> help(PyCintex.gbl.MyClass) # shows that normal python introspection works
+
+Other naming differences are such things as taking an address of an object.
+In PyCintex, this is done with ``AddressOf`` whereas in cppyy the choice was
+made to follow the naming as in ``ctypes`` and hence use ``addressof``
+(PyROOT/PyCintex predate ``ctypes`` by several years, and the ROOT project
+follows camel-case, hence the differences).
+
+Of course, this is python, so if any of the naming is not to your liking, all
+you have to do is provide a wrapper script that you import instead of
+importing the ``cppyy`` or ``PyCintex`` modules directly.
+In that wrapper script you can rename methods exactly the way you need it.
+
+In the cling world, all these differences will be resolved.
diff --git a/pypy/doc/extending.rst b/pypy/doc/extending.rst
--- a/pypy/doc/extending.rst
+++ b/pypy/doc/extending.rst
@@ -23,6 +23,8 @@
* Write them in RPython as mixedmodule_, using *rffi* as bindings.
+* Write them in C++ and bind them through Reflex_ (EXPERIMENTAL)
+
.. _ctypes: #CTypes
.. _\_ffi: #LibFFI
.. _mixedmodule: #Mixed Modules
@@ -110,3 +112,34 @@
XXX we should provide detailed docs about lltype and rffi, especially if we
want people to follow that way.
+
+Reflex
+======
+
+This method is only experimental for now, and is being exercised on a branch,
+`reflex-support`_, so you will have to build PyPy yourself.
+The method works by using the `Reflex package`_ to provide reflection
+information of the C++ code, which is then used to automatically generate
+bindings at runtime, which can then be used from python.
+Full details are `available here`_.
+
+.. _`reflex-support`: cppyy.html
+.. _`Reflex package`: http://root.cern.ch/drupal/content/reflex
+.. _`available here`: cppyy.html
+
+Pros
+----
+
+If it works, it is mostly automatic, and hence easy in use.
+The bindings can make use of direct pointers, in which case the calls are
+very fast.
+
+Cons
+----
+
+C++ is a large language, and these bindings are not yet feature-complete.
+Although missing features should do no harm if you don't use them, if you do
+need a particular feature, it may be necessary to work around it in python
+or with a C++ helper function.
+Although Reflex works on various platforms, the bindings with PyPy have only
+been tested on Linux.
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -110,6 +110,16 @@
space.call_method(w_list, "reverse")
return 0
+ at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject)
+def PyList_GetSlice(space, w_list, low, high):
+ """Return a list of the objects in list containing the objects between low
+ and high. Return NULL and set an exception if unsuccessful. Analogous
+ to list[low:high]. Negative indices, as when slicing from Python, are not
+ supported."""
+ w_start = space.wrap(low)
+ w_stop = space.wrap(high)
+ return space.getslice(w_list, w_start, w_stop)
+
@cpython_api([PyObject, Py_ssize_t, Py_ssize_t, PyObject], rffi.INT_real, error=-1)
def PyList_SetSlice(space, w_list, low, high, w_sequence):
"""Set the slice of list between low and high to the contents of
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -381,6 +381,15 @@
This is the equivalent of the Python expression hash(o)."""
return space.int_w(space.hash(w_obj))
+ at cpython_api([PyObject], lltype.Signed, error=-1)
+def PyObject_HashNotImplemented(space, o):
+ """Set a TypeError indicating that type(o) is not hashable and return -1.
+ This function receives special treatment when stored in a tp_hash slot,
+ allowing a type to explicitly indicate to the interpreter that it is not
+ hashable.
+ """
+ raise OperationError(space.w_TypeError, space.wrap("unhashable type"))
+
@cpython_api([PyObject], PyObject)
def PyObject_Dir(space, w_o):
"""This is equivalent to the Python expression dir(o), returning a (possibly
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -7,7 +7,7 @@
cpython_api, generic_cpy_call, PyObject, Py_ssize_t)
from pypy.module.cpyext.typeobjectdefs import (
unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc,
- getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc,
+ getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
readbufferproc)
@@ -60,6 +60,15 @@
args_w = space.fixedview(w_args)
return generic_cpy_call(space, func_binary, w_self, args_w[0])
+def wrap_inquirypred(space, w_self, w_args, func):
+ func_inquiry = rffi.cast(inquiry, func)
+ check_num_args(space, w_args, 0)
+ args_w = space.fixedview(w_args)
+ res = generic_cpy_call(space, func_inquiry, w_self)
+ if res == -1:
+ space.fromcache(State).check_and_raise_exception()
+ return space.wrap(bool(res))
+
def wrap_getattr(space, w_self, w_args, func):
func_target = rffi.cast(getattrfunc, func)
check_num_args(space, w_args, 1)
diff --git a/pypy/module/cpyext/stringobject.py b/pypy/module/cpyext/stringobject.py
--- a/pypy/module/cpyext/stringobject.py
+++ b/pypy/module/cpyext/stringobject.py
@@ -294,6 +294,26 @@
w_errors = space.wrap(rffi.charp2str(errors))
return space.call_method(w_str, 'encode', w_encoding, w_errors)
+ at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP], PyObject)
+def PyString_AsDecodedObject(space, w_str, encoding, errors):
+ """Decode a string object by passing it to the codec registered
+ for encoding and return the result as Python object. encoding and
+ errors have the same meaning as the parameters of the same name in
+ the string encode() method. The codec to be used is looked up
+ using the Python codec registry. Return NULL if an exception was
+ raised by the codec.
+
+ This function is not available in 3.x and does not have a PyBytes alias."""
+ if not PyString_Check(space, w_str):
+ PyErr_BadArgument(space)
+
+ w_encoding = w_errors = space.w_None
+ if encoding:
+ w_encoding = space.wrap(rffi.charp2str(encoding))
+ if errors:
+ w_errors = space.wrap(rffi.charp2str(errors))
+ return space.call_method(w_str, "decode", w_encoding, w_errors)
+
@cpython_api([PyObject, PyObject], PyObject)
def _PyString_Join(space, w_sep, w_seq):
return space.call_method(w_sep, 'join', w_seq)
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -1405,17 +1405,6 @@
"""
raise NotImplementedError
- at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject)
-def PyList_GetSlice(space, list, low, high):
- """Return a list of the objects in list containing the objects between low
- and high. Return NULL and set an exception if unsuccessful. Analogous
- to list[low:high]. Negative indices, as when slicing from Python, are not
- supported.
-
- This function used an int for low and high. This might
- require changes in your code for properly supporting 64-bit systems."""
- raise NotImplementedError
-
@cpython_api([Py_ssize_t], PyObject)
def PyLong_FromSsize_t(space, v):
"""Return a new PyLongObject object from a C Py_ssize_t, or
@@ -1606,15 +1595,6 @@
for PyObject_Str()."""
raise NotImplementedError
- at cpython_api([PyObject], lltype.Signed, error=-1)
-def PyObject_HashNotImplemented(space, o):
- """Set a TypeError indicating that type(o) is not hashable and return -1.
- This function receives special treatment when stored in a tp_hash slot,
- allowing a type to explicitly indicate to the interpreter that it is not
- hashable.
- """
- raise NotImplementedError
-
@cpython_api([], PyFrameObject)
def PyEval_GetFrame(space):
"""Return the current thread state's frame, which is NULL if no frame is
@@ -1737,17 +1717,6 @@
changes in your code for properly supporting 64-bit systems."""
raise NotImplementedError
- at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP], PyObject)
-def PyString_AsDecodedObject(space, str, encoding, errors):
- """Decode a string object by passing it to the codec registered for encoding and
- return the result as Python object. encoding and errors have the same
- meaning as the parameters of the same name in the string encode() method.
- The codec to be used is looked up using the Python codec registry. Return NULL
- if an exception was raised by the codec.
-
- This function is not available in 3.x and does not have a PyBytes alias."""
- raise NotImplementedError
-
@cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.CCHARP], PyObject)
def PyString_Encode(space, s, size, encoding, errors):
"""Encode the char buffer of the given size by passing it to the codec
diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -58,6 +58,11 @@
w_t = api.PyList_AsTuple(w_l)
assert space.unwrap(w_t) == (3, 2, 1)
+ def test_list_getslice(self, space, api):
+ w_l = space.newlist([space.wrap(3), space.wrap(2), space.wrap(1)])
+ w_s = api.PyList_GetSlice(w_l, 1, 5)
+ assert space.unwrap(w_s) == [2, 1]
+
class AppTestListObject(AppTestCpythonExtensionBase):
def test_listobject(self):
import sys
diff --git a/pypy/module/cpyext/test/test_stringobject.py b/pypy/module/cpyext/test/test_stringobject.py
--- a/pypy/module/cpyext/test/test_stringobject.py
+++ b/pypy/module/cpyext/test/test_stringobject.py
@@ -307,6 +307,13 @@
space.wrap(2), lltype.nullptr(rffi.CCHARP.TO), lltype.nullptr(rffi.CCHARP.TO)
)
+ def test_AsDecodedObject(self, space, api):
+ w_str = space.wrap('caf\xe9')
+ encoding = rffi.str2charp("latin-1")
+ w_res = api.PyString_AsDecodedObject(w_str, encoding, None)
+ rffi.free_charp(encoding)
+ assert space.unwrap(w_res) == u"caf\xe9"
+
def test_eq(self, space, api):
assert 1 == api._PyString_Eq(space.wrap("hello"), space.wrap("hello"))
assert 0 == api._PyString_Eq(space.wrap("hello"), space.wrap("world"))
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -488,3 +488,55 @@
assert type(it) is type(iter([]))
assert module.tp_iternext(it) == 1
raises(StopIteration, module.tp_iternext, it)
+
+ def test_bool(self):
+ module = self.import_extension('foo', [
+ ("newInt", "METH_VARARGS",
+ """
+ IntLikeObject *intObj;
+ long intval;
+ PyObject *name;
+
+ if (!PyArg_ParseTuple(args, "i", &intval))
+ return NULL;
+
+ IntLike_Type.tp_as_number = &intlike_as_number;
+ intlike_as_number.nb_nonzero = intlike_nb_nonzero;
+ if (PyType_Ready(&IntLike_Type) < 0) return NULL;
+ intObj = PyObject_New(IntLikeObject, &IntLike_Type);
+ if (!intObj) {
+ return NULL;
+ }
+
+ intObj->value = intval;
+ return (PyObject *)intObj;
+ """)],
+ """
+ typedef struct
+ {
+ PyObject_HEAD
+ int value;
+ } IntLikeObject;
+
+ static int
+ intlike_nb_nonzero(IntLikeObject *v)
+ {
+ if (v->value == -42) {
+ PyErr_SetNone(PyExc_ValueError);
+ return -1;
+ }
+ return v->value;
+ }
+
+ PyTypeObject IntLike_Type = {
+ PyObject_HEAD_INIT(0)
+ /*ob_size*/ 0,
+ /*tp_name*/ "IntLike",
+ /*tp_basicsize*/ sizeof(IntLikeObject),
+ };
+ static PyNumberMethods intlike_as_number;
+ """)
+ assert not bool(module.newInt(0))
+ assert bool(module.newInt(1))
+ assert bool(module.newInt(-1))
+ raises(ValueError, bool, module.newInt(-42))
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -5,6 +5,7 @@
interpleveldefs = {
'debug_repr': 'interp_extras.debug_repr',
'remove_invalidates': 'interp_extras.remove_invalidates',
+ 'set_invalidation': 'interp_extras.set_invalidation',
}
appleveldefs = {}
@@ -30,6 +31,7 @@
'isna': 'interp_numarray.isna',
'concatenate': 'interp_numarray.concatenate',
'repeat': 'interp_numarray.repeat',
+ 'where': 'interp_arrayops.where',
'set_string_function': 'appbridge.set_string_function',
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -10,6 +10,7 @@
from pypy.module.micronumpy.interp_dtype import get_dtype_cache
from pypy.module.micronumpy.interp_numarray import (Scalar, BaseArray,
scalar_w, W_NDimArray, array)
+from pypy.module.micronumpy.interp_arrayops import where
from pypy.module.micronumpy import interp_ufuncs
from pypy.rlib.objectmodel import specialize, instantiate
@@ -35,6 +36,7 @@
SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any",
"unegative", "flat", "tostring"]
TWO_ARG_FUNCTIONS = ["dot", 'take']
+THREE_ARG_FUNCTIONS = ['where']
class FakeSpace(object):
w_ValueError = None
@@ -445,14 +447,25 @@
arg = self.args[1].execute(interp)
if not isinstance(arg, BaseArray):
raise ArgumentNotAnArray
- if not isinstance(arg, BaseArray):
- raise ArgumentNotAnArray
if self.name == "dot":
w_res = arr.descr_dot(interp.space, arg)
elif self.name == 'take':
w_res = arr.descr_take(interp.space, arg)
else:
assert False # unreachable code
+ elif self.name in THREE_ARG_FUNCTIONS:
+ if len(self.args) != 3:
+ raise ArgumentMismatch
+ arg1 = self.args[1].execute(interp)
+ arg2 = self.args[2].execute(interp)
+ if not isinstance(arg1, BaseArray):
+ raise ArgumentNotAnArray
+ if not isinstance(arg2, BaseArray):
+ raise ArgumentNotAnArray
+ if self.name == "where":
+ w_res = where(interp.space, arr, arg1, arg2)
+ else:
+ assert False
else:
raise WrongFunctionName
if isinstance(w_res, BaseArray):
diff --git a/pypy/module/micronumpy/interp_arrayops.py b/pypy/module/micronumpy/interp_arrayops.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/interp_arrayops.py
@@ -0,0 +1,90 @@
+
+from pypy.module.micronumpy.interp_numarray import convert_to_array,\
+ VirtualArray
+from pypy.module.micronumpy import signature
+
+class WhereArray(VirtualArray):
+ def __init__(self, space, arr, x, y):
+ self.arr = arr
+ self.x = x
+ self.y = y
+ VirtualArray.__init__(self, 'where', arr.shape[:],
+ x.find_dtype())
+
+ def create_sig(self):
+ if self.forced_result is not None:
+ return self.forced_result.create_sig()
+ return signature.WhereSignature(self.res_dtype, self.arr.find_dtype(),
+ self.arr.create_sig(),
+ self.x.create_sig(),
+ self.y.create_sig())
+
+ def _del_sources(self):
+ self.arr = None
+ self.x = None
+ self.y = None
+
+def where(space, w_arr, w_x, w_y):
+ """where(condition, [x, y])
+
+ Return elements, either from `x` or `y`, depending on `condition`.
+
+ If only `condition` is given, return ``condition.nonzero()``.
+
+ Parameters
+ ----------
+ condition : array_like, bool
+ When True, yield `x`, otherwise yield `y`.
+ x, y : array_like, optional
+ Values from which to choose. `x` and `y` need to have the same
+ shape as `condition`.
+
+ Returns
+ -------
+ out : ndarray or tuple of ndarrays
+ If both `x` and `y` are specified, the output array contains
+ elements of `x` where `condition` is True, and elements from
+ `y` elsewhere.
+
+ If only `condition` is given, return the tuple
+ ``condition.nonzero()``, the indices where `condition` is True.
+
+ See Also
+ --------
+ nonzero, choose
+
+ Notes
+ -----
+ If `x` and `y` are given and input arrays are 1-D, `where` is
+ equivalent to::
+
+ [xv if c else yv for (c,xv,yv) in zip(condition,x,y)]
+
+ Examples
+ --------
+ >>> np.where([[True, False], [True, True]],
+ ... [[1, 2], [3, 4]],
+ ... [[9, 8], [7, 6]])
+ array([[1, 8],
+ [3, 4]])
+
+ >>> np.where([[0, 1], [1, 0]])
+ (array([0, 1]), array([1, 0]))
+
+ >>> x = np.arange(9.).reshape(3, 3)
+ >>> np.where( x > 5 )
+ (array([2, 2, 2]), array([0, 1, 2]))
+ >>> x[np.where( x > 3.0 )] # Note: result is 1D.
+ array([ 4., 5., 6., 7., 8.])
+ >>> np.where(x < 5, x, -1) # Note: broadcasting.
+ array([[ 0., 1., 2.],
+ [ 3., 4., -1.],
+ [-1., -1., -1.]])
+
+
+ NOTE: support for not passing x and y is unsupported
+ """
+ arr = convert_to_array(space, w_arr)
+ x = convert_to_array(space, w_x)
+ y = convert_to_array(space, w_y)
+ return WhereArray(space, arr, x, y)
diff --git a/pypy/module/micronumpy/interp_extras.py b/pypy/module/micronumpy/interp_extras.py
--- a/pypy/module/micronumpy/interp_extras.py
+++ b/pypy/module/micronumpy/interp_extras.py
@@ -1,5 +1,5 @@
from pypy.interpreter.gateway import unwrap_spec
-from pypy.module.micronumpy.interp_numarray import BaseArray
+from pypy.module.micronumpy.interp_numarray import BaseArray, get_numarray_cache
@unwrap_spec(array=BaseArray)
@@ -13,3 +13,7 @@
"""
del array.invalidates[:]
return space.w_None
+
+ at unwrap_spec(arg=bool)
+def set_invalidation(space, arg):
+ get_numarray_cache(space).enable_invalidation = arg
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -72,9 +72,10 @@
arr.force_if_needed()
del self.invalidates[:]
- def add_invalidates(self, other):
- self.invalidates.append(other)
-
+ def add_invalidates(self, space, other):
+ if get_numarray_cache(space).enable_invalidation:
+ self.invalidates.append(other)
+
def descr__new__(space, w_subtype, w_size, w_dtype=None):
dtype = space.interp_w(interp_dtype.W_Dtype,
space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
@@ -1583,3 +1584,10 @@
arr.fill(space, space.wrap(False))
return arr
return space.wrap(False)
+
+class NumArrayCache(object):
+ def __init__(self, space):
+ self.enable_invalidation = True
+
+def get_numarray_cache(space):
+ return space.fromcache(NumArrayCache)
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -278,7 +278,7 @@
else:
w_res = Call1(self.func, self.name, w_obj.shape, calc_dtype,
res_dtype, w_obj)
- w_obj.add_invalidates(w_res)
+ w_obj.add_invalidates(space, w_res)
return w_res
@@ -347,8 +347,8 @@
w_res = Call2(self.func, self.name,
new_shape, calc_dtype,
res_dtype, w_lhs, w_rhs, out)
- w_lhs.add_invalidates(w_res)
- w_rhs.add_invalidates(w_res)
+ w_lhs.add_invalidates(space, w_res)
+ w_rhs.add_invalidates(space, w_res)
if out:
w_res.get_concrete()
return w_res
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -498,3 +498,63 @@
arr.left.setitem(iterator.offset, value)
def debug_repr(self):
return 'AxisReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
+
+class WhereSignature(Signature):
+ _immutable_fields_ = ['dtype', 'arrdtype', 'arrsig', 'xsig', 'ysig']
+
+ def __init__(self, dtype, arrdtype, arrsig, xsig, ysig):
+ self.dtype = dtype
+ self.arrdtype = arrdtype
+ self.arrsig = arrsig
+ self.xsig = xsig
+ self.ysig = ysig
+
+ def hash(self):
+ return (intmask(self.arrsig.hash() << 1) ^
+ intmask(self.xsig.hash() << 2) ^
+ intmask(self.ysig.hash() << 3))
+
+ def eq(self, other, compare_array_no=True):
+ if type(self) is not type(other):
+ return False
+ assert isinstance(other, WhereSignature)
+ return (self.arrsig.eq(other.arrsig, compare_array_no) and
+ self.xsig.eq(other.xsig, compare_array_no) and
+ self.ysig.eq(other.ysig, compare_array_no))
+
+ def _invent_array_numbering(self, arr, cache):
+ from pypy.module.micronumpy.interp_arrayops import WhereArray
+ assert isinstance(arr, WhereArray)
+ self.arrsig._invent_array_numbering(arr.arr, cache)
+ self.xsig._invent_array_numbering(arr.x, cache)
+ self.ysig._invent_array_numbering(arr.y, cache)
+
+ def _invent_numbering(self, cache, allnumbers):
+ self.arrsig._invent_numbering(cache, allnumbers)
+ self.xsig._invent_numbering(cache, allnumbers)
+ self.ysig._invent_numbering(cache, allnumbers)
+
+ def _create_iter(self, iterlist, arraylist, arr, transforms):
+ from pypy.module.micronumpy.interp_arrayops import WhereArray
+
+ assert isinstance(arr, WhereArray)
+ # XXX this does not support broadcasting correctly
+ self.arrsig._create_iter(iterlist, arraylist, arr.arr, transforms)
+ self.xsig._create_iter(iterlist, arraylist, arr.x, transforms)
+ self.ysig._create_iter(iterlist, arraylist, arr.y, transforms)
+
+ def eval(self, frame, arr):
+ from pypy.module.micronumpy.interp_arrayops import WhereArray
+ assert isinstance(arr, WhereArray)
+ lhs = self.xsig.eval(frame, arr.x).convert_to(self.dtype)
+ rhs = self.ysig.eval(frame, arr.y).convert_to(self.dtype)
+ w_val = self.arrsig.eval(frame, arr.arr)
+ if self.arrdtype.itemtype.bool(w_val):
+ return lhs
+ else:
+ return rhs
+
+ def debug_repr(self):
+ return 'Where(%s, %s, %s)' % (self.arrsig.debug_repr(),
+ self.xsig.debug_repr(),
+ self.ysig.debug_repr())
diff --git a/pypy/module/micronumpy/test/test_arrayops.py b/pypy/module/micronumpy/test/test_arrayops.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_arrayops.py
@@ -0,0 +1,16 @@
+
+from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+
+class AppTestNumSupport(BaseNumpyAppTest):
+ def test_where(self):
+ from _numpypy import where, ones, zeros, array
+ a = [1, 2, 3, 0, -3]
+ a = where(array(a) > 0, ones(5), zeros(5))
+ assert (a == [1, 1, 1, 0, 0]).all()
+
+ def test_where_invalidates(self):
+ from _numpypy import where, ones, zeros, array
+ a = array([1, 2, 3, 0, -3])
+ b = where(a > 0, ones(5), zeros(5))
+ a[0] = 0
+ assert (b == [1, 1, 1, 0, 0]).all()
diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py
--- a/pypy/module/micronumpy/test/test_compile.py
+++ b/pypy/module/micronumpy/test/test_compile.py
@@ -270,3 +270,13 @@
b -> 2
""")
assert interp.results[0].value == 3
+
+ def test_where(self):
+ interp = self.run('''
+ a = [1, 0, 3, 0]
+ b = [1, 1, 1, 1]
+ c = [0, 0, 0, 0]
+ d = where(a, b, c)
+ d -> 1
+ ''')
+ assert interp.results[0].value == 0
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1,9 +1,8 @@
import py
-from pypy.conftest import gettestobjspace, option
+from pypy.conftest import option
from pypy.interpreter.error import OperationError
-from pypy.module.micronumpy import signature
from pypy.module.micronumpy.appbridge import get_appbridge_cache
from pypy.module.micronumpy.interp_iter import Chunk, Chunks
from pypy.module.micronumpy.interp_numarray import W_NDimArray, shape_agreement
@@ -1831,6 +1830,12 @@
a[a & 1 == 1] = array([8, 9, 10])
assert (a == [[0, 8], [2, 9], [4, 10]]).all()
+ def test_array_indexing_bool_setitem_multidim(self):
+ from _numpypy import arange
+ a = arange(10).reshape(5, 2)
+ a[a & 1 == 0] = 15
+ assert (a == [[15, 1], [15, 3], [15, 5], [15, 7], [15, 9]]).all()
+
def test_copy_kwarg(self):
from _numpypy import array
x = array([1, 2, 3])
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -1,7 +1,6 @@
from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
-
class AppTestUfuncs(BaseNumpyAppTest):
def test_ufunc_instance(self):
from _numpypy import add, ufunc
@@ -148,12 +147,13 @@
assert math.isnan(fmax(nan, 0))
assert math.isnan(fmax(0, nan))
assert math.isnan(fmax(nan, nan))
- # The numpy docs specify that
- # the FIRST NaN should be used if both are NaN
- # use copysign on both sides to sidestep bug in nan representaion
+ # The numpy docs specify that the FIRST NaN should be used if both are NaN
+ # Since comparisons with nnan and nan all return false,
+ # use copysign on both sides to sidestep bug in nan representaion
# on Microsoft win32
assert math.copysign(1., fmax(nnan, nan)) == math.copysign(1., nnan)
+
def test_fmin(self):
from _numpypy import fmin
import math
@@ -166,9 +166,9 @@
assert (fmin(a, [1]*5) == [ninf, -5, 0, 1, 1]).all()
assert math.isnan(fmin(nan, 0))
assert math.isnan(fmin(0, nan))
- # The numpy docs specify that
- # the FIRST NaN should be used if both are NaN
- # use copysign on both sides to sidestep bug in nan representaion
+ assert math.isnan(fmin(nan, nan))
+ # The numpy docs specify that the FIRST NaN should be used if both are NaN
+ # use copysign on both sides to sidestep bug in nan representaion
# on Microsoft win32
assert math.copysign(1., fmin(nnan, nan)) == math.copysign(1., nnan)
diff --git a/pypy/rlib/rbigint.py b/pypy/rlib/rbigint.py
--- a/pypy/rlib/rbigint.py
+++ b/pypy/rlib/rbigint.py
@@ -40,7 +40,7 @@
# In that case, do 5 bits at a time. The potential drawback is that
# a table of 2**5 intermediate results is computed.
-## FIVEARY_CUTOFF = 8 disabled for now
+FIVEARY_CUTOFF = 8
def _mask_digit(x):
@@ -456,7 +456,7 @@
# python adaptation: moved macros REDUCE(X) and MULT(X, Y, result)
# into helper function result = _help_mult(x, y, c)
- if 1: ## b.numdigits() <= FIVEARY_CUTOFF:
+ if b.numdigits() <= FIVEARY_CUTOFF:
# Left-to-right binary exponentiation (HAC Algorithm 14.79)
# http://www.cacr.math.uwaterloo.ca/hac/about/chap14.pdf
i = b.numdigits() - 1
@@ -469,30 +469,51 @@
z = _help_mult(z, a, c)
j >>= 1
i -= 1
-## else:
-## This code is disabled for now, because it assumes that
-## SHIFT is a multiple of 5. It could be fixed but it looks
-## like it's more troubles than benefits...
-##
-## # Left-to-right 5-ary exponentiation (HAC Algorithm 14.82)
-## # This is only useful in the case where c != None.
-## # z still holds 1L
-## table = [z] * 32
-## table[0] = z
-## for i in range(1, 32):
-## table[i] = _help_mult(table[i-1], a, c)
-## i = b.numdigits() - 1
-## while i >= 0:
-## bi = b.digit(i)
-## j = SHIFT - 5
-## while j >= 0:
-## index = (bi >> j) & 0x1f
-## for k in range(5):
-## z = _help_mult(z, z, c)
-## if index:
-## z = _help_mult(z, table[index], c)
-## j -= 5
-## i -= 1
+ else:
+ # Left-to-right 5-ary exponentiation (HAC Algorithm 14.82)
+ # This is only useful in the case where c != None.
+ # z still holds 1L
+ table = [z] * 32
+ table[0] = z
+ for i in range(1, 32):
+ table[i] = _help_mult(table[i-1], a, c)
+ i = b.numdigits()
+ # Note that here SHIFT is not a multiple of 5. The difficulty
+ # is to extract 5 bits at a time from 'b', starting from the
+ # most significant digits, so that at the end of the algorithm
+ # it falls exactly to zero.
+ # m = max number of bits = i * SHIFT
+ # m+ = m rounded up to the next multiple of 5
+ # j = (m+) % SHIFT = (m+) - (i * SHIFT)
+ # (computed without doing "i * SHIFT", which might overflow)
+ j = i % 5
+ if j != 0:
+ j = 5 - j
+ if not we_are_translated():
+ assert j == (i*SHIFT+4)//5*5 - i*SHIFT
+ #
+ accum = r_uint(0)
+ while True:
+ j -= 5
+ if j >= 0:
+ index = (accum >> j) & 0x1f
+ else:
+ # 'accum' does not have enough digit.
+ # must get the next digit from 'b' in order to complete
+ i -= 1
+ if i < 0:
+ break # done
+ bi = b.udigit(i)
+ index = ((accum << (-j)) | (bi >> (j+SHIFT))) & 0x1f
+ accum = bi
+ j += SHIFT
+ #
+ for k in range(5):
+ z = _help_mult(z, z, c)
+ if index:
+ z = _help_mult(z, table[index], c)
+ #
+ assert j == -5
if negativeOutput and z.sign != 0:
z = z.sub(c)
diff --git a/pypy/rlib/test/test_rbigint.py b/pypy/rlib/test/test_rbigint.py
--- a/pypy/rlib/test/test_rbigint.py
+++ b/pypy/rlib/test/test_rbigint.py
@@ -379,6 +379,18 @@
for n, expected in [(37, 9), (1291, 931), (67889, 39464)]:
v = two.pow(t, rbigint.fromint(n))
assert v.toint() == expected
+ #
+ # more tests, comparing against CPython's answer
+ enabled = sample(range(5*32), 10)
+ for i in range(5*32):
+ t = t.mul(two) # add one random bit
+ if random() >= 0.5:
+ t = t.add(rbigint.fromint(1))
+ if i not in enabled:
+ continue # don't take forever
+ n = randint(1, sys.maxint)
+ v = two.pow(t, rbigint.fromint(n))
+ assert v.toint() == pow(2, t.tolong(), n)
def test_pow_lln(self):
x = 10L
diff --git a/pypy/translator/c/src/exception.h b/pypy/translator/c/src/exception.h
--- a/pypy/translator/c/src/exception.h
+++ b/pypy/translator/c/src/exception.h
@@ -43,6 +43,16 @@
filename, lineno, functionname);
}
#endif
+#else /* !DO_LOG_EXC: define the function anyway, so that we can shut
+ off the prints of a debug_exc by remaking only testing_1.o */
+void RPyDebugReturnShowException(const char *msg, const char *filename,
+ long lineno, const char *functionname);
+#ifndef PYPY_NOT_MAIN_FILE
+void RPyDebugReturnShowException(const char *msg, const char *filename,
+ long lineno, const char *functionname)
+{
+}
+#endif
#endif /* DO_LOG_EXC */
/* Hint: functions and macros not defined here, like RPyRaiseException,
More information about the pypy-commit
mailing list