From richiehindle at users.sourceforge.net Mon Nov 14 23:28:38 2005 From: richiehindle at users.sourceforge.net (Richie Hindle) Date: Mon, 14 Nov 2005 23:28:38 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes/resources README.txt, NONE, 1.1 Message-ID: <20051114222838.1D2271E4009@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes/resources In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv15683 Added Files: README.txt Log Message: Added a README noting that you need resourcepackages for changes to the 'resources' directory to take effect. --- NEW FILE: README.txt --- When running SpamBayes from source, you need to install resourcepackage from http://resourcepackage.sourceforge.net/ in order for your changes to anything in the "resource" directory to take effect. Download ResourcePackage-1.0.0.tar.gz, unpack it into a temporary area, cd to that area, and run "python setup.py install". You can then delete the unpacked files. From richiehindle at users.sourceforge.net Mon Nov 14 23:39:16 2005 From: richiehindle at users.sourceforge.net (Richie Hindle) Date: Mon, 14 Nov 2005 23:39:16 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes Options.py,1.124,1.125 Message-ID: <20051114223916.9280E1E4010@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv18669 Modified Files: Options.py Log Message: The web training interface now defaults to Discard for both ham and spam. This should help to prevent database bloat, overtraining, and unbalanced training. Thanks to Michael Logies for the suggestion. The mailing list thread is here: http://mail.python.org/pipermail/spambayes/2005-October/018227.html Index: Options.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v retrieving revision 1.124 retrieving revision 1.125 diff -C2 -d -r1.124 -r1.125 *** Options.py 22 May 2005 03:33:15 -0000 1.124 --- Options.py 14 Nov 2005 22:39:13 -0000 1.125 *************** *** 925,929 **** BOOLEAN, RESTORE), ! ("default_ham_action", _("Default training for ham"), _("ham"), _("""When presented with the review list in the web interface, which button would you like checked by default when the message --- 925,929 ---- BOOLEAN, RESTORE), ! ("default_ham_action", _("Default training for ham"), _("discard"), _("""When presented with the review list in the web interface, which button would you like checked by default when the message *************** *** 931,935 **** (_("ham"), _("spam"), _("discard"), _("defer")), RESTORE), ! ("default_spam_action", _("Default training for spam"), _("spam"), _("""When presented with the review list in the web interface, which button would you like checked by default when the message --- 931,935 ---- (_("ham"), _("spam"), _("discard"), _("defer")), RESTORE), ! ("default_spam_action", _("Default training for spam"), _("discard"), _("""When presented with the review list in the web interface, which button would you like checked by default when the message From anadelonbrin at users.sourceforge.net Tue Nov 15 01:01:56 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Tue, 15 Nov 2005 01:01:56 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes Corpus.py,1.24,1.25 Message-ID: <20051115000156.537501E4009@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv2957/spambayes Modified Files: Corpus.py Log Message: Stop hiding KeyErrors. Add __contains__ to corpus objects. Enhance expiry so that it only iterates through the corpus if necessary. Index: Corpus.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/Corpus.py,v retrieving revision 1.24 retrieving revision 1.25 diff -C2 -d -r1.24 -r1.25 *** Corpus.py 9 May 2005 06:30:23 -0000 1.24 --- Corpus.py 15 Nov 2005 00:01:52 -0000 1.25 *************** *** 138,142 **** key = message.key() if options["globals", "verbose"]: ! print 'removing message %s from corpus' % (key) self.unCacheMessage(key) del self.msgs[key] --- 138,142 ---- key = message.key() if options["globals", "verbose"]: ! print 'removing message %s from corpus' % (key,) self.unCacheMessage(key) del self.msgs[key] *************** *** 153,157 **** if options["globals", "verbose"]: ! print 'placing %s in corpus cache' % (key) self.msgs[key] = message --- 153,157 ---- if options["globals", "verbose"]: ! print 'placing %s in corpus cache' % (key,) self.msgs[key] = message *************** *** 170,174 **** if options["globals", "verbose"]: ! print 'Flushing %s from corpus cache' % (key) try: --- 170,174 ---- if options["globals", "verbose"]: ! print 'Flushing %s from corpus cache' % (key,) try: *************** *** 185,188 **** --- 185,190 ---- msg = fromcorpus[key] msg.load() # ensure that the substance has been loaded + # Remove needs to be first, because add changes the directory + # of the message, and so remove won't work then. fromcorpus.removeMessage(msg) self.addMessage(msg) *************** *** 196,200 **** def __getitem__(self, key): '''Corpus is a dictionary''' ! amsg = self.msgs.get(key) if amsg is None: --- 198,205 ---- def __getitem__(self, key): '''Corpus is a dictionary''' ! amsg = self.msgs.get(key, "") ! ! if amsg == "": ! raise KeyError(key) if amsg is None: *************** *** 208,218 **** return self.msgs.keys() def __iter__(self): '''Corpus is iterable''' for key in self.keys(): ! try: ! yield self[key] ! except KeyError: ! pass def __str__(self): --- 213,223 ---- return self.msgs.keys() + def __contains__(self, other): + return other in self.msgs + def __iter__(self): '''Corpus is iterable''' for key in self.keys(): ! yield self[key] def __str__(self): *************** *** 237,252 **** def __init__(self, expireBefore): - '''Constructor''' self.expireBefore = expireBefore def removeExpiredMessages(self): '''Kill expired messages''' ! for msg in self: ! if msg.createTimestamp() < time.time() - self.expireBefore: if options["globals", "verbose"]: ! print 'message %s has expired' % (msg.key()) from spambayes.storage import NO_TRAINING_FLAG self.removeMessage(msg, observer_flags=NO_TRAINING_FLAG) --- 242,270 ---- def __init__(self, expireBefore): self.expireBefore = expireBefore + # Only check for expiry after this time. + self.expiry_due = time.time() def removeExpiredMessages(self): '''Kill expired messages''' + + # Only check for expired messages after this time. We set this to the + # closest-to-expiry message's expiry time, so that this method can be + # called very regularly, and most of the time it will just immediately + # return. + if time.time() < self.expiry_due: + return ! self.expiry_due = time.time() + self.expireBefore ! for key in self.keys()[:]: ! msg = self[key] ! timestamp = msg.createTimestamp() ! if timestamp < time.time() - self.expireBefore: if options["globals", "verbose"]: ! print 'message %s has expired' % (msg.key(),) from spambayes.storage import NO_TRAINING_FLAG self.removeMessage(msg, observer_flags=NO_TRAINING_FLAG) + elif timestamp + self.expireBefore < self.expiry_due: + self.expiry_due = timestamp + self.expireBefore From anadelonbrin at users.sourceforge.net Tue Nov 15 01:13:57 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Tue, 15 Nov 2005 01:13:57 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes UserInterface.py, 1.58, 1.59 Message-ID: <20051115001357.83C0B1E4009@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv5262/spambayes Modified Files: UserInterface.py Log Message: Add missing import. Enter value for app even in subclass. Index: UserInterface.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/UserInterface.py,v retrieving revision 1.58 retrieving revision 1.59 diff -C2 -d -r1.58 -r1.59 *** UserInterface.py 22 May 2005 03:32:19 -0000 1.58 --- UserInterface.py 15 Nov 2005 00:13:54 -0000 1.59 *************** *** 84,88 **** import PyMeldLite - import Version import Dibbler import tokenizer --- 84,87 ---- *************** *** 140,144 **** htmlSource, self._images = self.readUIResources() self.html = PyMeldLite.Meld(htmlSource, readonly=True) ! self.app_for_version = None def onIncomingConnection(self, clientSocket): --- 139,143 ---- htmlSource, self._images = self.readUIResources() self.html = PyMeldLite.Meld(htmlSource, readonly=True) ! self.app_for_version = "SpamBayes" def onIncomingConnection(self, clientSocket): From anadelonbrin at users.sourceforge.net Tue Nov 15 01:16:23 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Tue, 15 Nov 2005 01:16:23 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes tokenizer.py,1.36,1.37 Message-ID: <20051115001623.DA35D1E4009@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv5584/spambayes Modified Files: tokenizer.py Log Message: Make it easier to subclass and use options in testing. Index: tokenizer.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/tokenizer.py,v retrieving revision 1.36 retrieving revision 1.37 diff -C2 -d -r1.36 -r1.37 *** tokenizer.py 26 May 2005 01:38:42 -0000 1.36 --- tokenizer.py 15 Nov 2005 00:16:20 -0000 1.37 *************** *** 1224,1227 **** --- 1224,1235 ---- def __init__(self): + self.setup() + + def setup(self): + """Get the tokenizer ready to use; this should be called after + all options have been set.""" + # We put this here, rather than in __init__, so that this can be + # done after we set options at runtime (since the tokenizer + # instance is generally created when this module is imported). if options["Tokenizer", "basic_header_tokenize"]: self.basic_skip = [re.compile(s) *************** *** 1609,1611 **** yield t ! tokenize = Tokenizer().tokenize --- 1617,1620 ---- yield t ! global_tokenizer = Tokenizer() ! tokenize = global_tokenizer.tokenize From anadelonbrin at users.sourceforge.net Tue Nov 15 01:22:19 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Tue, 15 Nov 2005 01:22:19 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes storage.py,1.51,1.52 Message-ID: <20051115002219.3BFA31E4009@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv6579/spambayes Modified Files: storage.py Log Message: Allow subclassing of ZODB storage. Work with earlier versions of ZODB. Add simple handling of ConflictErrors. Pack database. Have onAddMessage honour training flags. Index: storage.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v retrieving revision 1.51 retrieving revision 1.52 diff -C2 -d -r1.51 -r1.52 *** storage.py 22 Apr 2005 04:08:25 -0000 1.51 --- storage.py 15 Nov 2005 00:22:15 -0000 1.52 *************** *** 65,68 **** --- 65,69 ---- import os import sys + import time import types from spambayes import classifier *************** *** 677,681 **** from persistent import Persistent except ImportError: ! Persistent = object class _PersistentClassifier(classifier.Classifier, Persistent): def __init__(self): --- 678,685 ---- from persistent import Persistent except ImportError: ! try: ! from ZODB import Persistent ! except ImportError: ! Persistent = object class _PersistentClassifier(classifier.Classifier, Persistent): def __init__(self): *************** *** 687,690 **** --- 691,697 ---- class ZODBClassifier(object): + # Allow subclasses to override classifier class. + ClassifierClass = _PersistentClassifier + def __init__(self, db_name): self.db_filename = db_name *************** *** 702,706 **** def __setattr__(self, att, value): # For some attributes, we change the classifier instead. ! if att in ["nham", "nspam"]: setattr(self.classifier, att, value) else: --- 709,713 ---- def __setattr__(self, att, value): # For some attributes, we change the classifier instead. ! if att in ("nham", "nspam") and hasattr(self, "classifier"): setattr(self.classifier, att, value) else: *************** *** 729,733 **** self.conn = self.db.open() root = self.conn.root() ! self.classifier = root.get(self.db_name) if self.classifier is None: --- 736,740 ---- self.conn = self.db.open() root = self.conn.root() ! self.classifier = root.get(self.db_name) if self.classifier is None: *************** *** 735,739 **** if options["globals", "verbose"]: print >> sys.stderr, self.db_name, 'is a new ZODB' ! self.classifier = root[self.db_name] = _PersistentClassifier() else: if options["globals", "verbose"]: --- 742,746 ---- if options["globals", "verbose"]: print >> sys.stderr, self.db_name, 'is a new ZODB' ! self.classifier = root[self.db_name] = self.ClassifierClass() else: if options["globals", "verbose"]: *************** *** 742,750 **** self.nspam) self.closed = False ! def store(self): '''Place state into persistent store''' ! import ZODB ! import transaction assert self.closed == False, "Can't store a closed database" --- 749,764 ---- self.nspam) self.closed = False ! def store(self): '''Place state into persistent store''' ! try: ! import ZODB ! import ZODB.Transaction ! except ImportError: ! import transaction ! commit = transaction.commit ! else: ! commit = ZODB.Transaction.get_transaction().commit ! from ZODB.POSException import ConflictError assert self.closed == False, "Can't store a closed database" *************** *** 753,757 **** print >> sys.stderr, 'Persisting', self.db_name, 'state in database' ! transaction.commit() def close(self): --- 767,778 ---- print >> sys.stderr, 'Persisting', self.db_name, 'state in database' ! try: ! commit() ! except ConflictError: ! # We'll save it next time, or on close. It'll be lost if we ! # hard-crash, but that's unlikely, and not a particularly big ! # deal. ! if options["globals", "verbose"]: ! print >> sys.stderr, "Conflict on commit", self.db_name def close(self): *************** *** 765,769 **** --- 786,803 ---- # Do the closing. self.db.close() + + # We don't make any use of the 'undo' capabilities of the + # FileStorage at the moment, so might as well pack the database + # each time it is closed, to save as much disk space as possible. + # Pack it up to where it was 'yesterday'. + # XXX What is the 'referencesf' parameter for pack()? It doesn't + # XXX seem to do anything according to the source. + if hasattr(self.storage, "pack"): + self.storage.pack(time.time()-60*60*24, None) self.storage.close() + + # Ensure that we cannot continue to use this classifier. + delattr(self, "classifier") + self.closed = True if options["globals", "verbose"]: *************** *** 812,819 **** def onAddMessage(self, message, flags=0): '''A message is being added to an observed corpus.''' ! # There are no flags that we currently care about, so ! # get rid of the variable so that PyChecker doesn't bother us. ! del flags ! self.train(message) def train(self, message): --- 846,851 ---- def onAddMessage(self, message, flags=0): '''A message is being added to an observed corpus.''' ! if not (flags & NO_TRAINING_FLAG): ! self.train(message) def train(self, message): From anadelonbrin at users.sourceforge.net Tue Nov 15 01:28:10 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Tue, 15 Nov 2005 01:28:10 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes OptionsClass.py, 1.28, 1.29 Message-ID: <20051115002810.4E1321E4009@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv7260/spambayes Modified Files: OptionsClass.py Log Message: Handle setting notate options to nothing. Index: OptionsClass.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/OptionsClass.py,v retrieving revision 1.28 retrieving revision 1.29 diff -C2 -d -r1.28 -r1.29 *** OptionsClass.py 22 May 2005 03:32:56 -0000 1.28 --- OptionsClass.py 15 Nov 2005 00:28:07 -0000 1.29 *************** *** 689,693 **** self.get("Headers", "header_unsure_string")) ! return val in header_strings if self.is_valid(sect, opt, val): self._options[sect, opt.lower()].set(val) --- 689,693 ---- self.get("Headers", "header_unsure_string")) ! return val in header_strings or not val if self.is_valid(sect, opt, val): self._options[sect, opt.lower()].set(val) From anadelonbrin at users.sourceforge.net Tue Nov 15 01:30:38 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Tue, 15 Nov 2005 01:30:38 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes Options.py,1.125,1.126 Message-ID: <20051115003038.17BE41E4009@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv7716/spambayes Modified Files: Options.py Log Message: Handle setting notate options to none. Fix typo. Index: Options.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v retrieving revision 1.125 retrieving revision 1.126 diff -C2 -d -r1.125 -r1.126 *** Options.py 14 Nov 2005 22:39:13 -0000 1.125 --- Options.py 15 Nov 2005 00:30:34 -0000 1.126 *************** *** 734,743 **** spam messages; simply tick the boxes of the classifications take should be identified in this fashion."""), ! (_("ham"), _("spam"), _("unsure")), RESTORE), ("notate_subject", _("Classify in subject: header"), (), _("""This option will add the same information as 'Notate To', but to the start of the mail subject line."""), ! (_("ham"), _("spam"), _("unsure")), RESTORE), ), --- 734,743 ---- spam messages; simply tick the boxes of the classifications take should be identified in this fashion."""), ! ((), _("ham"), _("spam"), _("unsure")), RESTORE), ("notate_subject", _("Classify in subject: header"), (), _("""This option will add the same information as 'Notate To', but to the start of the mail subject line."""), ! ((), _("ham"), _("spam"), _("unsure")), RESTORE), ), *************** *** 782,786 **** ("retrieval_timeout", _("Retrieval timeout"), 30, ! _("""When proxying mesasges, time out after this length of time if all the headers have been received. The rest of the mesasge will proxy straight through. Some clients have a short timeout period, --- 782,786 ---- ("retrieval_timeout", _("Retrieval timeout"), 30, ! _("""When proxying messages, time out after this length of time if all the headers have been received. The rest of the mesasge will proxy straight through. Some clients have a short timeout period, *************** *** 872,875 **** --- 872,923 ---- ), + # imap4proxy settings: The only mandatory option is imap4proxy_servers, eg. + # "imap4.my-isp.com:143", or a comma-separated list of those. The ":143" + # is optional. If you specify more than one server in imap4proxy_servers, + # you must specify the same number of ports in imap4proxy_ports. + "imap4proxy" : ( + ("remote_servers", _("Remote Servers"), (), + _("""The SpamBayes IMAP4 proxy intercepts incoming email and classifies + it before sending it on to your email client. You need to specify + which IMAP4 server(s) you wish it to intercept - a IMAP4 server + address typically looks like "mail.myisp.net". If you use more than + one server, simply separate their names with commas. You can get + these server names from your existing email configuration, or from + your ISP or system administrator. If you are using Web-based email, + you can't use the SpamBayes IMAP4 proxy (sorry!). In your email + client's configuration, where you would normally put your IMAP4 server + address, you should now put the address of the machine running + SpamBayes."""), + SERVER, DO_NOT_RESTORE), + + ("listen_ports", _("SpamBayes Ports"), (), + _("""Each IMAP4 server that is being monitored must be assigned to a + 'port' in the SpamBayes IMAP4 proxy. This port must be different for + each monitored server, and there must be a port for each monitored + server. Again, you need to configure your email client to use this + port. If there are multiple servers, you must specify the same number + of ports as servers, separated by commas. If you don't know what to + use here, and you only have one server, try 143, or if that doesn't + work, try 8143."""), + SERVER, DO_NOT_RESTORE), + + ("allow_remote_connections", _("Allowed remote IMAP4 connections"), "localhost", + _("""Enter a list of trusted IPs, separated by commas. Remote IMAP + connections from any of them will be allowed. You can trust any + IP using a single '*' as field value. You can also trust ranges of + IPs using the '*' character as a wildcard (for instance 192.168.0.*). + The localhost IP will always be trusted. Type 'localhost' in the + field to trust this only address."""), + IP_LIST, RESTORE), + + ("use_ssl", "Connect via a secure socket layer", False, + """Use SSL to connect to the server. This allows spambayes to connect + without sending data in plain text. + + Note that this does not check the server certificate at this point in + time.""", + (False, True, "automatic"), DO_NOT_RESTORE), + ), + "html_ui" : ( ("port", _("Port"), 8880, From anadelonbrin at users.sourceforge.net Tue Nov 15 01:36:02 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Tue, 15 Nov 2005 01:36:02 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes message.py,1.69,1.70 Message-ID: <20051115003602.A50B31E4009@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv8563/spambayes Modified Files: message.py Log Message: Add ZODB version for messageinfo class. Fix up the messed up messageinfo db property of a message business. Make it a bit easier to subclass SBHeaderMessage to add more headers. Fix a bug with adding the exception header. Index: message.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/message.py,v retrieving revision 1.69 retrieving revision 1.70 diff -C2 -d -r1.69 -r1.70 *** message.py 3 Jun 2005 03:34:42 -0000 1.69 --- message.py 15 Nov 2005 00:35:57 -0000 1.70 *************** *** 6,10 **** Message - an email.Message.Message, extended with spambayes methods SBHeaderMessage - A Message with spambayes header manipulations ! MessageInfoDB - persistent state storage for Message Abstract: --- 6,12 ---- Message - an email.Message.Message, extended with spambayes methods SBHeaderMessage - A Message with spambayes header manipulations ! MessageInfoDB - persistent state storage for Message, using dbm ! MessageInfoZODB - persistent state storage for Message, using ZODB ! MessageInfoPickle - persistent state storage for Message, using pickle Abstract: *************** *** 119,123 **** class MessageInfoBase(object): ! def __init__(self, db_name): self.db_name = db_name --- 121,125 ---- class MessageInfoBase(object): ! def __init__(self, db_name=None): self.db_name = db_name *************** *** 135,148 **** self.store() def load_msg(self, msg): if self.db is not None: try: try: ! attributes = self.db[msg.getDBKey()] except pickle.UnpicklingError: # The old-style Outlook message info db didn't use # shelve, so get it straight from the dbm. if hasattr(self, "dbm"): ! attributes = self.dbm[msg.getDBKey()] else: raise --- 137,158 ---- self.store() + def __getstate__(self): + return self.db + + def __setstate__(self, state): + self.db = state + def load_msg(self, msg): if self.db is not None: + key = msg.getDBKey() + assert key is not None, "None is not a valid key." try: try: ! attributes = self.db[key] except pickle.UnpicklingError: # The old-style Outlook message info db didn't use # shelve, so get it straight from the dbm. if hasattr(self, "dbm"): ! attributes = self.dbm[key] else: raise *************** *** 179,183 **** for att in msg.stored_attributes: attributes.append((att, getattr(msg, att))) ! self.db[msg.getDBKey()] = attributes self.store() --- 189,195 ---- for att in msg.stored_attributes: attributes.append((att, getattr(msg, att))) ! key = msg.getDBKey() ! assert key is not None, "None is not a valid key." ! self.db[key] = attributes self.store() *************** *** 247,250 **** --- 259,288 ---- self.db.sync() + # If ZODB isn't available, then this class won't be useable, but we + # still need to be able to import this module. So we pretend that all + # is ok. + try: + from persistent import Persistent + except ImportError: + Persistent = object + class _PersistentMessageInfo(MessageInfoBase, Persistent): + def __init__(self): + import ZODB + from BTrees.OOBTree import OOBTree + + MessageInfoBase.__init__(self) + self.db = OOBTree() + + class MessageInfoZODB(storage.ZODBClassifier): + ClassifierClass = _PersistentMessageInfo + def __init__(self, db_name): + self.nham = self.nspam = 0 # Only used for debugging prints + storage.ZODBClassifier.__init__(self, db_name) + self.classifier.store = self.store + def __setattr__(self, att, value): + # Override ZODBClassifier.__setattr__ + object.__setattr__(self, att, value) + + # values are classifier class, True if it accepts a mode # arg, and True if the argument is a pathname *************** *** 254,258 **** ## "mysql" : (MessageInfoMySQL, False, False), ## "cdb" : (MessageInfoCDB, False, True), ! ## "zodb" : (MessageInfoZODB, False, True), ## "zeo" : (MessageInfoZEO, False, False), } --- 292,296 ---- ## "mysql" : (MessageInfoMySQL, False, False), ## "cdb" : (MessageInfoCDB, False, True), ! "zodb" : (MessageInfoZODB, False, True), ## "zeo" : (MessageInfoZEO, False, False), } *************** *** 280,296 **** ! class Message(email.Message.Message): '''An email.Message.Message extended for SpamBayes''' ! def __init__(self, id=None, message_info_db=None): email.Message.Message.__init__(self) # persistent state # (non-persistent state includes all of email.Message.Message state) - if message_info_db is not None: - self.message_info_db = message_info_db - else: - nm, typ = database_type() - self.message_info_db = open_storage(nm, typ) self.stored_attributes = ['c', 't', 'date_modified', ] self.getDBKey = self.getId --- 318,329 ---- ! class Message(object, email.Message.Message): '''An email.Message.Message extended for SpamBayes''' ! def __init__(self, id=None): email.Message.Message.__init__(self) # persistent state # (non-persistent state includes all of email.Message.Message state) self.stored_attributes = ['c', 't', 'date_modified', ] self.getDBKey = self.getId *************** *** 303,306 **** --- 336,363 ---- self.setId(id) + # This whole message info database thing is a real mess. It really + # ought to be a property of the Message class, not each instance. + # So we want to access it via classmethods. However, we have treated + # it as a regular attribute, so need to make it a property. To make + # a classmethod property, we have to jump through some hoops, which we + # deserver for not doing it right in the first place. + _message_info_db = None + def _get_class_message_info_db(klass): + # If, the first time we access the attribute, it hasn't been + # set, then we load up the default one. + if klass._message_info_db is None: + nm, typ = database_type() + klass._message_info_db = open_storage(nm, typ) + return klass._message_info_db + _get_class_message_info_db = classmethod(_get_class_message_info_db) + def _set_class_message_info_db(klass, value): + klass._message_info_db = value + _set_class_message_info_db = classmethod(_set_class_message_info_db) + def _get_message_info_db(self): + return self._get_class_message_info_db() + def _set_message_info_db(self, value): + self._set_class_message_info_db(value) + message_info_db = property(_get_message_info_db, _set_message_info_db) + # This function (and it's hackishness) can be avoided by using # email.message_from_string(text, _class=SBHeaderMessage) *************** *** 332,336 **** def setId(self, id): if self.id and self.id != id: ! raise ValueError, "MsgId has already been set, cannot be changed" if id is None: --- 389,393 ---- def setId(self, id): if self.id and self.id != id: ! raise ValueError, "MsgId has already been set, cannot be changed" + `self.id` + `id` if id is None: *************** *** 440,447 **** return self.id ! def addSBHeaders(self, prob, clues): ! """Add hammie header, and remember message's classification. Also, ! add optional headers if needed.""" ! if prob < options['Categorization','ham_cutoff']: disposition = options['Headers','header_ham_string'] --- 497,501 ---- return self.id ! def setDisposition(self, prob): if prob < options['Categorization','ham_cutoff']: disposition = options['Headers','header_ham_string'] *************** *** 451,454 **** --- 505,514 ---- disposition = options['Headers','header_unsure_string'] self.RememberClassification(disposition) + + def addSBHeaders(self, prob, clues): + """Add hammie header, and remember message's classification. Also, + add optional headers if needed.""" + self.setDisposition(prob) + disposition = self.GetClassification() self[options['Headers','classification_header_name']] = disposition *************** *** 479,483 **** word = email.Header.Header(word, charset='utf-8').encode() ! evd.append("%r: %.2f" % (word, score)) # Line-wrap this header, because it can get very long. We don't --- 539,546 ---- word = email.Header.Header(word, charset='utf-8').encode() ! try: ! evd.append("%r: %.2f" % (word, score)) ! except TypeError: ! evd.append("%r: %s" % (word, score)) # Line-wrap this header, because it can get very long. We don't *************** *** 646,648 **** headers += "%s: %s\r\n" % \ (options["Headers", "mailid_header_name"], msg_id) ! return (headers + body, details) --- 709,711 ---- headers += "%s: %s\r\n" % \ (options["Headers", "mailid_header_name"], msg_id) ! return (headers + '\r\n' + body, details) From anadelonbrin at users.sourceforge.net Tue Nov 15 01:40:31 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Tue, 15 Nov 2005 01:40:31 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes Dibbler.py,1.16,1.17 Message-ID: <20051115004031.96F021E4009@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv9175/spambayes Modified Files: Dibbler.py Log Message: Add a few comvenience methods to Context. Index: Dibbler.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/Dibbler.py,v retrieving revision 1.16 retrieving revision 1.17 diff -C2 -d -r1.16 -r1.17 *** Dibbler.py 22 Dec 2004 03:32:19 -0000 1.16 --- Dibbler.py 15 Nov 2005 00:40:28 -0000 1.17 *************** *** 234,237 **** --- 234,243 ---- self._HTTPPort = None # Stores the port for `run(launchBrowser=True)` self._map = asyncMap + def pop(self, key): + return self._map.pop(key) + def keys(self): + return self._map.keys() + def __len__(self): + return len(self._map) _defaultContext = Context() From anadelonbrin at users.sourceforge.net Tue Nov 15 01:42:42 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Tue, 15 Nov 2005 01:42:42 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes FileCorpus.py,1.20,1.21 Message-ID: <20051115004242.B51611E400A@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv9451/spambayes Modified Files: FileCorpus.py Log Message: Handle flags on addMessage. Make it easier to subclass FileMessage. Don't shadow IOErrors. Index: FileCorpus.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/FileCorpus.py,v retrieving revision 1.20 retrieving revision 1.21 diff -C2 -d -r1.20 -r1.21 *** FileCorpus.py 22 Apr 2005 07:15:09 -0000 1.20 --- FileCorpus.py 15 Nov 2005 00:42:39 -0000 1.21 *************** *** 88,92 **** return msg ! def addMessage(self, message): '''Add a Message to this corpus''' if not fnmatch.fnmatch(message.key(), self.filter): --- 88,92 ---- return msg ! def addMessage(self, message, observer_flags=0): '''Add a Message to this corpus''' if not fnmatch.fnmatch(message.key(), self.filter): *************** *** 100,104 **** # superclass processing *MUST* be done # perform superclass processing *LAST!* ! Corpus.Corpus.addMessage(self, message) def removeMessage(self, message, observer_flags=0): --- 100,104 ---- # superclass processing *MUST* be done # perform superclass processing *LAST!* ! Corpus.Corpus.addMessage(self, message, observer_flags) def removeMessage(self, message, observer_flags=0): *************** *** 148,158 **** '''Message that persists as a file system artifact.''' ! def __init__(self, file_name=None, directory=None, ! message_database=None): '''Constructor(message file name, corpus directory name)''' self.file_name = file_name self.directory = directory self.loaded = False ! self._msg = message.SBHeaderMessage(message_info_db=message_database) def __getattr__(self, att): --- 148,158 ---- '''Message that persists as a file system artifact.''' ! message_class = message.SBHeaderMessage ! def __init__(self, file_name=None, directory=None): '''Constructor(message file name, corpus directory name)''' self.file_name = file_name self.directory = directory self.loaded = False ! self._msg = self.message_class() def __getattr__(self, att): *************** *** 211,215 **** try: self._msg = email.message_from_string(\ ! fp.read(), _class = message.SBHeaderMessage) except IOError, e: if str(e) == 'Not a gzipped file': --- 211,215 ---- try: self._msg = email.message_from_string(\ ! fp.read(), _class = self.message_class) except IOError, e: if str(e) == 'Not a gzipped file': *************** *** 219,224 **** fp = open(self.pathname(), 'rb') self._msg = email.message_from_string(\ ! fp.read(), _class = message.SBHeaderMessage) fp.close() else: fp.close() --- 219,227 ---- fp = open(self.pathname(), 'rb') self._msg = email.message_from_string(\ ! fp.read(), _class = self.message_class) fp.close() + else: + # Don't shadow other errors. + raise else: fp.close() *************** *** 299,306 **** # Subclass must define a concrete message klass. klass = None - def __init__(self, message_database=None): - self.message_database = message_database - Corpus.MessageFactory.__init__(self) - def create(self, key, directory, content=None): '''Create a message object from a filename in a directory''' --- 302,305 ---- *************** *** 312,316 **** msg.loaded = True return msg ! return self.klass(key, directory, self.message_database) --- 311,315 ---- msg.loaded = True return msg ! return self.klass(key, directory) From anadelonbrin at users.sourceforge.net Sat Nov 26 06:06:48 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Sat, 26 Nov 2005 06:06:48 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes oe_mailbox.py,1.10,1.11 Message-ID: <20051126050648.E88221E4007@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv2731/spambayes Modified Files: oe_mailbox.py Log Message: Missing self. Index: oe_mailbox.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/oe_mailbox.py,v retrieving revision 1.10 retrieving revision 1.11 diff -C2 -d -r1.10 -r1.11 *** oe_mailbox.py 23 Nov 2004 00:15:42 -0000 1.10 --- oe_mailbox.py 26 Nov 2005 05:06:40 -0000 1.11 *************** *** 213,217 **** self.__readValues(dbxStream, 0, dbxAddress, 0, dbxValues) ! def __readValues(self, dbxStream, dbxParent, dbxAddress, dbxPosition, dbxValues): dbxStream.seek(dbxAddress) dbxBuffer = dbxStream.read(dbxTree.TREE_NODE_SIZE) --- 213,217 ---- self.__readValues(dbxStream, 0, dbxAddress, 0, dbxValues) ! def __readValues(self, dbxStream, unused, dbxAddress, dbxPosition, unused2): dbxStream.seek(dbxAddress) dbxBuffer = dbxStream.read(dbxTree.TREE_NODE_SIZE) *************** *** 323,327 **** def getIndexDataType(self, dbxIndex): """Returns the data type of the given index.""" ! return DT_NONE def getValue(self, dbxIndex): --- 323,327 ---- def getIndexDataType(self, dbxIndex): """Returns the data type of the given index.""" ! return self.DT_NONE def getValue(self, dbxIndex): From anadelonbrin at users.sourceforge.net Sat Nov 26 06:07:40 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Sat, 26 Nov 2005 06:07:40 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes storage.py,1.52,1.53 Message-ID: <20051126050740.C39441E4007@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv3010/spambayes Modified Files: storage.py Log Message: Abort failed transactions. Index: storage.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v retrieving revision 1.52 retrieving revision 1.53 diff -C2 -d -r1.52 -r1.53 *** storage.py 15 Nov 2005 00:22:15 -0000 1.52 --- storage.py 26 Nov 2005 05:07:32 -0000 1.53 *************** *** 445,455 **** c.execute("select word from bayes") rows = self.fetchall(c) ! # There is probably some clever way to do this with map or ! # something, but I don't know what it is. We want the first ! # element from all the items in 'rows' ! keys = [] ! for r in rows: ! keys.append(r[0]) ! return keys --- 445,449 ---- c.execute("select word from bayes") rows = self.fetchall(c) ! return [r[0] for r in rows] *************** *** 758,764 **** import transaction commit = transaction.commit else: commit = ZODB.Transaction.get_transaction().commit ! from ZODB.POSException import ConflictError assert self.closed == False, "Can't store a closed database" --- 752,761 ---- import transaction commit = transaction.commit + abort = transaction.abort else: commit = ZODB.Transaction.get_transaction().commit ! abort = ZODB.Transaction.get_transaction().abort ! from ZODB.POSException import ConflictError ! from ZODB.POSException import TransactionFailedError assert self.closed == False, "Can't store a closed database" *************** *** 775,778 **** --- 772,782 ---- if options["globals", "verbose"]: print >> sys.stderr, "Conflict on commit", self.db_name + abort() + except TransactionFailedError: + # Saving isn't working. Try to abort, but chances are that + # restarting is needed. + print >> sys.stderr, "Storing failed. Need to restart.", \ + self.db_name + abort() def close(self): From anadelonbrin at users.sourceforge.net Sat Nov 26 06:12:58 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Sat, 26 Nov 2005 06:12:58 +0100 (CET) Subject: [Spambayes-checkins] spambayes/scripts sb_server.py,1.46,1.47 Message-ID: <20051126051258.03DBA1E4007@bag.python.org> Update of /cvsroot/spambayes/spambayes/scripts In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv4914/scripts Modified Files: sb_server.py Log Message: More handling of Python 2.4's stupid asyncore error handling. Add ability to use a different asyncore map. Fix closing of ssl sockets. Let callers change the name of the mutex. Fix message database code to match new, corrected, style. Index: sb_server.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/scripts/sb_server.py,v retrieving revision 1.46 retrieving revision 1.47 diff -C2 -d -r1.46 -r1.47 *** sb_server.py 22 Apr 2005 07:15:08 -0000 1.46 --- sb_server.py 26 Nov 2005 05:12:49 -0000 1.47 *************** *** 144,150 **** synchronously, because that would block the process.""" ! def __init__(self, serverName, serverPort, lineCallback, ssl=False): ! Dibbler.BrighterAsyncChat.__init__(self) self.lineCallback = lineCallback self.request = '' self.set_terminator('\r\n') --- 144,152 ---- synchronously, because that would block the process.""" ! def __init__(self, serverName, serverPort, lineCallback, ssl=False, ! map=None): ! Dibbler.BrighterAsyncChat.__init__(self, map=map) self.lineCallback = lineCallback + self.handled_exception = False self.request = '' self.set_terminator('\r\n') *************** *** 201,204 **** --- 203,213 ---- return self.ssl_socket.write(data) + def handle_expt(self): + # Python 2.4's system of continuously pumping error messages + # is stupid. Print an error once, and then ignore. + if not self.handled_exception: + print >> sys.stderr, "Unhandled exception in ServerLineReader" + self.handled_exception = True + def recv_ssl(self, buffer_size): try: *************** *** 231,235 **** self.close() try: ! del self.ssl_socket, self.socket except AttributeError: pass --- 240,244 ---- self.close() try: ! del self.ssl_socket except AttributeError: pass *************** *** 249,253 **** """ ! def __init__(self, clientSocket, serverName, serverPort, ssl=False): Dibbler.BrighterAsyncChat.__init__(self, clientSocket) self.request = '' --- 258,263 ---- """ ! def __init__(self, clientSocket, serverName, serverPort, ! ssl=False, map=Dibbler._defaultContext._map): Dibbler.BrighterAsyncChat.__init__(self, clientSocket) self.request = '' *************** *** 268,272 **** self.serverSocket = ServerLineReader(serverName, serverPort, ! self.onServerLine, ssl) def onIncomingConnection(self, clientSocket): --- 278,282 ---- self.serverSocket = ServerLineReader(serverName, serverPort, ! self.onServerLine, ssl, map) def onIncomingConnection(self, clientSocket): *************** *** 530,534 **** """Adds the judgement header based on the raw headers and body of the message.""" ! # Previous, we used '\n\r?\n' to detect the end of the headers in # case of broken emails that don't use the proper line separators, # and if we couldn't find it, then we assumed that the response was --- 540,544 ---- """Adds the judgement header based on the raw headers and body of the message.""" ! # Previously, we used '\n\r?\n' to detect the end of the headers in # case of broken emails that don't use the proper line separators, # and if we couldn't find it, then we assumed that the response was *************** *** 653,659 **** # multiple servers starting at once. Platform specific as no reasonable # cross-platform solution exists (however, an old trick is to use a ! # directory for a mutex, as a "create/test" atomic API generally exists. # Will return a handle to be later closed, or may throw AlreadyRunningException ! def open_platform_mutex(): if sys.platform.startswith("win"): try: --- 663,669 ---- # multiple servers starting at once. Platform specific as no reasonable # cross-platform solution exists (however, an old trick is to use a ! # directory for a mutex, as a "create/test" atomic API generally exists). # Will return a handle to be later closed, or may throw AlreadyRunningException ! def open_platform_mutex(mutex_name="SpamBayesServer"): if sys.platform.startswith("win"): try: *************** *** 670,674 **** # "SpamBayesServer" mutex, if for no better reason than so # an installer can check if we are running - mutex_name = "SpamBayesServer" try: hmutex = win32event.CreateMutex(None, True, mutex_name) --- 680,683 ---- *************** *** 889,895 **** map(storage.ensureDir, [sc, hc, uc]) if self.gzipCache: ! factory = GzipFileMessageFactory(self.mdb) else: ! factory = FileMessageFactory(self.mdb) age = options["Storage", "cache_expiry_days"]*24*60*60 self.spamCorpus = ExpiryFileCorpus(age, factory, sc, --- 898,904 ---- map(storage.ensureDir, [sc, hc, uc]) if self.gzipCache: ! factory = GzipFileMessageFactory() else: ! factory = FileMessageFactory() age = options["Storage", "cache_expiry_days"]*24*60*60 self.spamCorpus = ExpiryFileCorpus(age, factory, sc, From anadelonbrin at users.sourceforge.net Sat Nov 26 06:33:11 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Sat, 26 Nov 2005 06:33:11 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes Options.py, 1.126, 1.127 storage.py, 1.53, 1.54 Message-ID: <20051126053311.925EF1E4007@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv11374/spambayes Modified Files: Options.py storage.py Log Message: Remove 1.0 backwards compatibility options of using True for dbm and False for pickle. Add read-only mode to ZODBClassifier. Index: Options.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v retrieving revision 1.126 retrieving revision 1.127 diff -C2 -d -r1.126 -r1.127 *** Options.py 15 Nov 2005 00:30:34 -0000 1.126 --- Options.py 26 Nov 2005 05:33:03 -0000 1.127 *************** *** 525,530 **** (i.e. changing "True" to "dbm" and "False" to "pickle", or sticking with the default."""), ! # True == "dbm", False == "pickle", "True" == "dbm", "False" == "pickle" ! ("zeo", "zodb", "cdb", "mysql", "pgsql", "dbm", "pickle", "True", "False", True, False), RESTORE), ("persistent_storage_file", _("Storage file name"), "hammie.db", --- 525,529 ---- (i.e. changing "True" to "dbm" and "False" to "pickle", or sticking with the default."""), ! ("zeo", "zodb", "cdb", "mysql", "pgsql", "dbm", "pickle"), RESTORE), ("persistent_storage_file", _("Storage file name"), "hammie.db", Index: storage.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v retrieving revision 1.53 retrieving revision 1.54 diff -C2 -d -r1.53 -r1.54 *** storage.py 26 Nov 2005 05:07:32 -0000 1.53 --- storage.py 26 Nov 2005 05:33:03 -0000 1.54 *************** *** 688,695 **** ClassifierClass = _PersistentClassifier ! def __init__(self, db_name): self.db_filename = db_name self.db_name = os.path.basename(db_name) self.closed = True self.load() --- 688,696 ---- ClassifierClass = _PersistentClassifier ! def __init__(self, db_name, mode='c'): self.db_filename = db_name self.db_name = os.path.basename(db_name) self.closed = True + self.mode = mode self.load() *************** *** 711,715 **** import ZODB from ZODB.FileStorage import FileStorage ! self.storage = FileStorage(self.db_filename) def load(self): --- 712,717 ---- import ZODB from ZODB.FileStorage import FileStorage ! self.storage = FileStorage(self.db_filename, ! read_only=self.mode=='r') def load(self): *************** *** 926,930 **** "mysql" : (mySQLClassifier, False, False), "cdb" : (CDBClassifier, False, True), ! "zodb" : (ZODBClassifier, False, True), "zeo" : (ZEOClassifier, False, False), } --- 928,932 ---- "mysql" : (mySQLClassifier, False, False), "cdb" : (CDBClassifier, False, True), ! "zodb" : (ZODBClassifier, True, True), "zeo" : (ZEOClassifier, False, False), } *************** *** 990,998 **** if nm is None and typ is None: typ = options[default_type] - # Backwards compatibility crud. - if typ is True or typ == "True": - typ = "dbm" - elif typ is False or typ == "False": - typ = "pickle" try: unused, unused, is_path = _storage_types[typ] --- 992,995 ---- From anadelonbrin at users.sourceforge.net Sat Nov 26 06:35:18 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Sat, 26 Nov 2005 06:35:18 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes Options.py,1.127,1.128 Message-ID: <20051126053518.80C691E4009@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv12088/spambayes Modified Files: Options.py Log Message: Change default storage to ZODB, as per spambayes-dev discussion a couple of weeks back. Index: Options.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v retrieving revision 1.127 retrieving revision 1.128 diff -C2 -d -r1.127 -r1.128 *** Options.py 26 Nov 2005 05:33:03 -0000 1.127 --- Options.py 26 Nov 2005 05:35:10 -0000 1.128 *************** *** 516,528 **** # loaded by the appropriate application only. "Storage" : ( ! ("persistent_use_database", _("Use database for storage"), "dbm", ! _("""SpamBayes can use either a database (quick to score one message) ! or a pickle (quick to train on huge amounts of messages). There is ! also (currently experimental) the ability to use a mySQL or ! PostgrepSQL database. For historical reasons, if you set this to ! "True" you are selecting "dbm" and if you set this to "False" you ! are selecting "pickle". We recommend explicitly selecting the type, ! (i.e. changing "True" to "dbm" and "False" to "pickle", or sticking ! with the default."""), ("zeo", "zodb", "cdb", "mysql", "pgsql", "dbm", "pickle"), RESTORE), --- 516,524 ---- # loaded by the appropriate application only. "Storage" : ( ! ("persistent_use_database", _("Use database for storage"), "zodb", ! _("""SpamBayes can use either a ZODB or bsddb database (quick to score ! one message) or a pickle (quick to train on huge amounts of messages). ! There is also (currently experimental) the ability to use a mySQL or ! PostgrepSQL database."""), ("zeo", "zodb", "cdb", "mysql", "pgsql", "dbm", "pickle"), RESTORE), From anadelonbrin at users.sourceforge.net Sat Nov 26 07:44:23 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Sat, 26 Nov 2005 07:44:23 +0100 (CET) Subject: [Spambayes-checkins] spambayes/utilities convert_db.py,NONE,1.1 Message-ID: <20051126064423.B9A851E4009@bag.python.org> Update of /cvsroot/spambayes/spambayes/utilities In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv29992/utilities Added Files: convert_db.py Log Message: To make the conversion from dbm to ZODB easier for users: (a) change the names to hammie.fs and messageinfo.fs (b) add a conversion utility to automatically convert the database. --- NEW FILE: convert_db.py --- #! /usr/bin/env python """convert_db.py Simplified version of sb_dbimpexp.py to aid with the change from defaulting to dbm in 1.0.x to zodb in 1.1.x. Usage: sb_dbexpimp [options] options: -t type : type of the database to convert (e.g. pickle, dbm, zodb) -T type : type of database to convert to (e.g. pickle, dbm, zodb) -n path : path to the database to convert -N path : path of the resulting database -h : help To convert the database from dbm to ZODB on Windows, simply running the script with no options should work. To convert the database from dbm to ZODB on linux or OS X, the following should work: python convert_db.py -n ~/.hammie.db """ # This module is part of the spambayes project, which is Copyright 2002-5 # The Python Software Foundation and is covered by the Python Software # Foundation license. __author__ = "Tony Meyer " __credits__ = "Tim Stone; all the SpamBayes folk" import os import sys import getopt from spambayes import storage if __name__ == '__main__': try: opts, args = getopt.getopt(sys.argv[1:], 'ht:T:n:N:') except getopt.error, msg: print >>sys.stderr, str(msg) + '\n\n' + __doc__ sys.exit() old_name = old_type = new_name = new_type = None for opt, arg in opts: if opt == '-h': print >>sys.stderr, __doc__ sys.exit() elif opt == '-t': old_type = arg elif opt == '-T': new_type = arg elif opt == '-n': old_name = os.path.expanduser(arg) elif opt == '-N': new_name = os.path.expanduser(arg) storage.convert(old_name, old_type, new_name, new_type) From anadelonbrin at users.sourceforge.net Sat Nov 26 07:44:23 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Sat, 26 Nov 2005 07:44:23 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes Options.py, 1.128, 1.129 storage.py, 1.54, 1.55 Message-ID: <20051126064423.B8E591E4007@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv29992/spambayes Modified Files: Options.py storage.py Log Message: To make the conversion from dbm to ZODB easier for users: (a) change the names to hammie.fs and messageinfo.fs (b) add a conversion utility to automatically convert the database. Index: Options.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v retrieving revision 1.128 retrieving revision 1.129 diff -C2 -d -r1.128 -r1.129 *** Options.py 26 Nov 2005 05:35:10 -0000 1.128 --- Options.py 26 Nov 2005 06:44:15 -0000 1.129 *************** *** 523,527 **** ("zeo", "zodb", "cdb", "mysql", "pgsql", "dbm", "pickle"), RESTORE), ! ("persistent_storage_file", _("Storage file name"), "hammie.db", _("""Spambayes builds a database of information that it gathers from incoming emails and from you, the user, to get better and --- 523,527 ---- ("zeo", "zodb", "cdb", "mysql", "pgsql", "dbm", "pickle"), RESTORE), ! ("persistent_storage_file", _("Storage file name"), "hammie.fs", _("""Spambayes builds a database of information that it gathers from incoming emails and from you, the user, to get better and *************** *** 532,536 **** FILE_WITH_PATH, DO_NOT_RESTORE), ! ("messageinfo_storage_file", _("Message information file name"), "spambayes.messageinfo.db", _("""Spambayes builds a database of information about messages that it has already seen and trained or classified. This --- 532,536 ---- FILE_WITH_PATH, DO_NOT_RESTORE), ! ("messageinfo_storage_file", _("Message information file name"), "messageinfo.fs", _("""Spambayes builds a database of information about messages that it has already seen and trained or classified. This Index: storage.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v retrieving revision 1.54 retrieving revision 1.55 diff -C2 -d -r1.54 -r1.55 *** storage.py 26 Nov 2005 05:33:03 -0000 1.54 --- storage.py 26 Nov 2005 06:44:15 -0000 1.55 *************** *** 1002,1005 **** --- 1002,1038 ---- return nm, typ + def convert(old_name=None, old_type=None, new_name=None, new_type=None): + # The expected need is to convert the existing hammie.db dbm + # database to a hammie.fs ZODB database. + if old_name is None: + old_name = "hammie.db" + if old_type is None: + old_type = "dbm" + if new_name is None or new_type is None: + auto_name, auto_type = database_type({}) + if new_name is None: + new_name = auto_name + if new_type is None: + new_type = auto_type + + old_bayes = open_storage(old_name, old_type) + new_bayes = open_storage(new_name, new_type) + words = old_bayes._wordinfokeys() + + new_bayes.nham = old_bayes.nham + new_bayes.nspam = old_bayes.nspam + + print >> sys.stderr, "Converting %s (%s database) to " \ + "%s (%s database)." % (old_name, old_type, new_name, new_type) + print >> sys.stderr, "Database has %s ham, %s spam, and %s words." % \ + (new_bayes.nham, new_bayes.nspam, len(words)) + + for word in words: + new_bayes._wordinfoset(word, old_bayes._wordinfoget(word)) + + print "Storing database, please be patient..." + new_bayes.store() + print "Conversion complete." + def ensureDir(dirname): """Ensure that the given directory exists - in other words, if it From anadelonbrin at users.sourceforge.net Sun Nov 27 01:42:19 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Sun, 27 Nov 2005 01:42:19 +0100 (CET) Subject: [Spambayes-checkins] spambayes/windows/py2exe setup_all.py, 1.24, 1.25 Message-ID: <20051127004219.076FE1E400F@bag.python.org> Update of /cvsroot/spambayes/spambayes/windows/py2exe In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv3128/windows/py2exe Modified Files: setup_all.py Log Message: Create a little utility to convert the database from dbm to ZODB, install it, and offer to run it on install. Index: setup_all.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/windows/py2exe/setup_all.py,v retrieving revision 1.24 retrieving revision 1.25 diff -C2 -d -r1.24 -r1.25 *** setup_all.py 7 Apr 2005 05:57:47 -0000 1.24 --- setup_all.py 27 Nov 2005 00:42:11 -0000 1.25 *************** *** 127,130 **** --- 127,134 ---- script = os.path.join(sb_top_dir, "windows", "autoconfigure.py"), ) + convert = dict( + dest_base = "bin/convert_database", + script = os.path.join(sb_top_dir, "utilities", "convert_db.py"), + ) outlook_data_files = [ *************** *** 176,182 **** # A service service=[service], ! # console exes for debugging console=[sb_server, sb_upload, outlook_dump_props, sb_pop3dnd, ! sb_imapfilter], # The taskbar windows=[pop3proxy_tray, outlook_addin_register, autoconfigure], --- 180,186 ---- # A service service=[service], ! # console exes console=[sb_server, sb_upload, outlook_dump_props, sb_pop3dnd, ! sb_imapfilter, convert], # The taskbar windows=[pop3proxy_tray, outlook_addin_register, autoconfigure], From anadelonbrin at users.sourceforge.net Sun Nov 27 01:42:19 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Sun, 27 Nov 2005 01:42:19 +0100 (CET) Subject: [Spambayes-checkins] spambayes/windows spambayes.iss,1.24,1.25 Message-ID: <20051127004219.7D9991E400F@bag.python.org> Update of /cvsroot/spambayes/spambayes/windows In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv3128/windows Modified Files: spambayes.iss Log Message: Create a little utility to convert the database from dbm to ZODB, install it, and offer to run it on install. Index: spambayes.iss =================================================================== RCS file: /cvsroot/spambayes/spambayes/windows/spambayes.iss,v retrieving revision 1.24 retrieving revision 1.25 diff -C2 -d -r1.24 -r1.25 *** spambayes.iss 7 Apr 2005 04:24:50 -0000 1.24 --- spambayes.iss 27 Nov 2005 00:42:11 -0000 1.25 *************** *** 49,52 **** --- 49,54 ---- Source: "py2exe\dist\bin\sb_imapfilter.exe"; DestDir: "{app}\bin"; Check: InstallingIMAP; Flags: ignoreversion + Source: "py2exe\dist\bin\convert_database.exe"; DestDir: "{app}\bin"; Flags: ignoreversion + ; There is a problem attempting to get Inno to unregister our DLL. If we mark our DLL ; as 'regserver', it installs and registers OK, but at uninstall time, it unregisters *************** *** 66,69 **** --- 68,72 ---- [Run] + FileName:"{app}\bin\convert_database.exe"; Description: "Convert the database from 1.0 to 1.1"; Flags: postinstall skipifdoesntexist; Check: ConvertDatabase FileName:"{app}\bin\sb_tray.exe"; Description: "Start the server now"; Flags: postinstall skipifdoesntexist nowait; Check: InstallingProxy *************** *** 118,121 **** --- 121,128 ---- Result := startup_imap; end; + function ConvertDatabase() : Boolean; + begin + Result := convert_db; + end; function IsOutlookInstalled() : Boolean; *************** *** 180,183 **** --- 187,191 ---- allusers := False; startup_imap := False; + convert_db := False; end; *************** *** 225,229 **** { Validate certain pages before allowing the user to proceed } if CurPageID = TasksPage.ID then begin ! I := 0; if InstallOutlook then begin allusers := TasksPage.Values[I]; --- 233,238 ---- { Validate certain pages before allowing the user to proceed } if CurPageID = TasksPage.ID then begin ! convert_db := TasksPage.Values[0]; ! I := 1; if InstallOutlook then begin allusers := TasksPage.Values[I]; *************** *** 282,285 **** --- 291,296 ---- 'Select the components you would like Setup to perform while installing SpamBayes, then click Next.', False, False); + TasksPage.Add('Convert 1.0 database to 1.1 format'); + TasksPage.Values[0] := True; if InstallOutlook then TasksPage.Add('Register add-in for all users'); *************** *** 310,313 **** --- 321,325 ---- S := S + 'Additional Tasks:' + NewLine; + if convert_db then S := S + Space + 'Convert database from 1.0 to 1.1 format' + NewLine if startup then S := S + Space + 'Run Proxy on Startup' + NewLine if desktop then S := S + Space + 'Install Proxy Desktop Icon' + NewLine From anadelonbrin at users.sourceforge.net Sun Nov 27 02:10:03 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Sun, 27 Nov 2005 02:10:03 +0100 (CET) Subject: [Spambayes-checkins] spambayes CHANGELOG.txt,1.52,1.53 Message-ID: <20051127011003.690611E4017@bag.python.org> Update of /cvsroot/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv8224 Modified Files: CHANGELOG.txt Log Message: Bring up to date. Index: CHANGELOG.txt =================================================================== RCS file: /cvsroot/spambayes/spambayes/CHANGELOG.txt,v retrieving revision 1.52 retrieving revision 1.53 diff -C2 -d -r1.52 -r1.53 *** CHANGELOG.txt 16 Mar 2005 03:19:44 -0000 1.52 --- CHANGELOG.txt 27 Nov 2005 01:09:54 -0000 1.53 *************** *** 1,4 **** --- 1,45 ---- [Note that all dates are in English, not American format - i.e. day/month/year] + Release 1.1a2 + ============= + Tony Meyer 27/11/2005 Install the conversion utility and offer to run it on Windows install. + Tony Meyer 26/11/2005 Add conversion utility to easily convert dbm to ZODB. + Tony Meyer 26/11/2005 Change database names to hammie.fs and messageinfo.fs + Tony Meyer 26/11/2005 Have ZODBClassifier honour read-only mode. + Tony Meyer 26/11/2005 Change default storage to ZODB. + Tony Meyer 26/11/2005 Remove backwards compatibility option to use True for dbm and False for pickle. + Tony Meyer 26/11/2005 Fix closing SSL connections in POP3 proxy. + Tony Meyer 26/11/2005 Abort failed ZODB transactions. + Tony Meyer 15/11/2005 Fix exception header adding. + Tony Meyer 15/11/2005 Tidy up the way that messages keep track of the messageinfo database. + Tony Meyer 15/11/2005 Add ZODB version of MessageInfo class. + Tony Meyer 15/11/2005 Let the notate_ options be turned off. + Tony Meyer 15/11/2005 Add simple handling of ZODB ConflictErrors. + Tony Meyer 15/11/2005 Pack ZODB database. + Tony Meyer 15/11/2005 Fix tokenizer so that the -o switch can be used with all options. + Tony Meyer 15/11/2005 Only run through expiry if messages might expire. + Richie Hindle 15/11/2005 Change default action for review page to discard for ham and spam. + Tony Meyer 08/07/2005 imapfilter: If a folder (e.g. ham_folder) isn't selected, then don't try and expunge it. + Sjoerd Mullender 03/07/2005 imapfilter: It seems it is possible that we get an invalid date string from self.extractTime(), so protect against that. + Mark Hammond 21/06/2005 Update Outlook folder dialogs so we work with new and old pywin32 versions. + Tony Meyer 03/06/2005 Use has_key() and not "in" for Python 2.2 compat. + Skip Montanaro 03/06/2005 sb_filter: If database doesn't exist, initialize it. Closes the ancient #759917. + Sjoerd Mullender 30/05/2005 imapfilter: After closing a folder, there is no current folder, so set current_folder to None, avoiding a crash on close. + Tony Meyer 26/05/2005 Use get_content_type instead of the deprecated get_type + Tony Meyer 22/05/2005 ProxyUI: Fix the handling of the notate_to and notate_subject options. + Sjoerd Mullender 19/05/2005 imapfilter: In each iteration, create a new IMAPSession object and hence a new connection to the server. + Tony Meyer 13/05/2005 Fix [ 1182754 ] 1.1a1: imapfilter browser chokes on incorrect password + Tony Meyer 26/04/2005 Add ability to handle unsures to fpfn.py + Tony Meyer 26/04/2005 Add [ 618932 ] fpfn.py: add interactivity on unix + Tony Meyer 22/04/2005 Fix [ 1182671 ] When cache directories are full, 1.1a1 starts slowly + Tony Meyer 22/04/2005 Fix [ 1187208 ] import into CDB chokes on 8-bit chars + Tony Meyer 21/04/2005 Use the os.path.basename as the database name for a ZODB FileStorage database, so that if the database is moved it will still work. + Tony Meyer 21/04/2005 Add [ 1182703 ] sb_imapfilter binary should output to log + Skip Montanaro 19/04/2005 tte.py: Include time to create & reverse mailbox in accounting for each round + Tony Meyer 14/04/2005 Fix [ 1179055 ] minor UI correction needed (1.1.a1) + Tony Meyer 14/04/2005 Highlight current row of review table with a little bit of javascript. + Tony Meyer 13/04/2005 Fix [ 1181160 ] Language fallback doesn't work + Tony Meyer 07/04/2005 Add basic options to sort+group.py + Release 1.1a1 ============= From anadelonbrin at users.sourceforge.net Sun Nov 27 03:15:41 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Sun, 27 Nov 2005 03:15:41 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes Options.py,1.129,1.130 Message-ID: <20051127021541.E02771E4007@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv17716/spambayes Modified Files: Options.py Log Message: Make a start on 1.1a2. Index: Options.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v retrieving revision 1.129 retrieving revision 1.130 diff -C2 -d -r1.129 -r1.130 *** Options.py 26 Nov 2005 06:44:15 -0000 1.129 --- Options.py 27 Nov 2005 02:15:33 -0000 1.130 *************** *** 516,521 **** # loaded by the appropriate application only. "Storage" : ( ! ("persistent_use_database", _("Use database for storage"), "zodb", ! _("""SpamBayes can use either a ZODB or bsddb database (quick to score one message) or a pickle (quick to train on huge amounts of messages). There is also (currently experimental) the ability to use a mySQL or --- 516,521 ---- # loaded by the appropriate application only. "Storage" : ( ! ("persistent_use_database", _("Database backend"), "zodb", ! _("""SpamBayes can use either a ZODB or dbm database (quick to score one message) or a pickle (quick to train on huge amounts of messages). There is also (currently experimental) the ability to use a mySQL or From anadelonbrin at users.sourceforge.net Sun Nov 27 03:15:42 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Sun, 27 Nov 2005 03:15:42 +0100 (CET) Subject: [Spambayes-checkins] spambayes WHAT_IS_NEW.txt,1.39,1.40 Message-ID: <20051127021542.2FBEB1E4007@bag.python.org> Update of /cvsroot/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv17716 Modified Files: WHAT_IS_NEW.txt Log Message: Make a start on 1.1a2. Index: WHAT_IS_NEW.txt =================================================================== RCS file: /cvsroot/spambayes/spambayes/WHAT_IS_NEW.txt,v retrieving revision 1.39 retrieving revision 1.40 diff -C2 -d -r1.39 -r1.40 *** WHAT_IS_NEW.txt 24 May 2005 23:41:58 -0000 1.39 --- WHAT_IS_NEW.txt 27 Nov 2005 02:15:33 -0000 1.40 *************** *** 16,20 **** is released. ! New in 1.1 Alpha 1 ================== --- 16,20 ---- is released. ! New in 1.1 Alpha 2 ================== *************** *** 24,29 **** -------------------------------------------- ! There should be no incompatible changes (from 1.0.2) in this release. ! If you are transitioning from a version older than 1.0.2, please read the notes in the previous release notes (accessible from ). --- 24,61 ---- -------------------------------------------- ! SpamBayes has changed to use ZODB as the default database backend, rather ! than dbm (usually bsddb). There are three methods for handling this ! transition: ! ! o You can start with a fresh database. It's a good idea to start ! training over occasionally, and SpamBayes learns very rapidly, so this ! is a feasible option. This is the default behaviour, unless you have ! set SpamBayes to use a different name or backend type, in which case ! your choices will remain unchanged. ! ! o You can continue to use your existing database files. To do this, you ! need to set SpamBayes to use "dbm" as the persistent storage method ! in the configuration. POP3 Proxy and IMAP4 filter users can do this ! via the configuration page; Outlook and command-line tool users need ! to manually add these lines to their configuration file: ! [Storage] ! persistent_use_database:dbm ! ! o You can convert your existing database files to the new format. ! Windows users will be given the opportunity to do this on installation; ! other users should use the utilities/convert_db.py script to do this. ! Note that only the token database (containing your training) is ! converted; the 'messageinfo' database (containing statistics about ! SpamBayes performance and a record of which messages have been ! filtered and trained) will be replaced with an empty one. The only ! noticeable effect of this will be that IMAP Filter users will have ! all messages refiltered and the Statistics reports will be restarted. ! ! If you have changed your configuration to specify which backend to use in ! the past, note that the deprecated values of "True" (meaning "dbm") and ! "False" (meaning "pickle") are no longer supported (use "dbm" or "pickle"). ! ! There should be no other incompatible changes (from 1.0.4) in this release. ! If you are transitioning from a version older than 1.0.4, please read the notes in the previous release notes (accessible from ). *************** *** 36,170 **** General ------- ! o Localisations (translations of the user interface) of SpamBayes are ! now available. 1.1 includes French and Spanish translations - many ! thanks to the translators for their efforts. If you would like to ! add a translation in a new language (or dialect), please contact the ! SpamBayes development team at spambayes-dev at python.org. For Windows ! users, SpamBayes will use the same language as the operating system ! is set to use (if available). This can be overridden (or set, for ! non-Windows users) via the configuration file. ! o The version number scheme has been improved and simplified. ! o Many more unittests have been added. For the end user, this should ! mean that changes in later versions are much less likely to break ! existing functionality. ! o New storage types are available: CDBClassifier (uses a CDB database, ! similar to that used by CdbClassifier, but not identical), ! ZODBClassifier (uses a ZODB FileStorage object) and ZEOClassifier ! (connects to a ZEO server). ! o The "use bigrams" option is no longer experimental, although is still ! off by default. Outlook Plugin -------------- ! o The Manager dialog is enlarged (at least 800x600 resolution is needed ! to view it - 640x480 users will need to continue with 1.0.2) and ! enhanced to (among other things) include a new "Statistics" tab. ! o The "Show Clues" message is enchanced. This now includes information ! about the last time the message was trained or classified. ! o For messages received from a local Exchange server, which have no ! Internet headers, we now generate more (and better) headers from the ! available message properties. ! o Statistics are vastly improved. More statistics are reported, and ! these are persistent across Outlook sessions. The statistics may ! be 'reset' so that they are calculated from a specified date and time. ! o Enable movement/copying of ham (good) messages, like the existing ! functionality for unsure/spam messages. ! o The "Delete As Spam" button is now simply "Spam", and the ! "Recover From Spam" button simply "Not Spam". The behaviour of ! the buttons has not changed. You may need to recreate the toolbar ! to see these changes (right click on the toolbar, click Customize, ! select SpamBayes, click Delete, then restart Outlook). ! o Notification sounds can now be set, via the Manager dialog. ! o The SpamBayes menu now includes an "Empty Spam Folder" option. ! o Outlook users can now use a variety of different underlying database ! storage methods, in the same fashion as non-Outlook users. ! POP3 Proxy (sb_server.py) ------------------------- ! o POP over SSL is partially available. The connection from the mail ! client to sb_server still cannot be over SSL, but the connection ! from sb_server to the mail server can be. The latter is more likely ! to be at risk (as it is more likely to be non-local), so is the ! more important, as well as the easier to implement. ! o "Subject" (and "To") notations are removed from messages before display ! in the review pages and before training such messages. ! o The "To" header notations are in a new form, which is correctly an ! email address. ! o A particular type of malformed message (one with no header-body ! separator) would pass through SpamBayes without classification, or ! a X-Spambayes-Exception header. This is no longer the case. ! o If you run sb_server as a Windows service messages are now logged ! to the Windows Event log rather than to SpamBayes log files. ! o Installation of the Windows service for sb_server has been improved, ! and a better data directory is now used (if a configuration file ! does not already exist, then a "SpamBayesData" directory in the ! directory of the application is used). Web interface (sb_server.py and sb_imapfilter.py) ------------------------------------------------- ! o Improve the "More Statistics" page. More statistics are reported, and ! the statistics may be 'reset' so that they are calculated from a ! specified date and time. ! o A new option "Use bigrams" is available from the Advanced Configuration ! page. ! o Messages that are uploaded (from the "Upload" box on the main page, ! or via sb_upload.py) are now properly added to the cache directories, ! and _pop3proxyham.mbox and pop3proxy_spam.mbox are no longer created. ! o Various minor fixes and improvements. ! IMAP Filter (sb_imapfilter.py) ------------------------------ ! o Use the "Message-ID" header as the SpamBayes ID (when possible). This ! means that the IMAP filter should not need to rewrite nearly as many ! messages (hopefully, should need to rewrite no ham messages), and should ! also work faster and more reliably. ! o A ham folder may now be specified, like the spam and unsure folders. ! o Continue on (skipping) if a folder is unable to be acccesed for training ! or filtering. ! o Filtering multiple servers with a single sb_imapfilter process is now ! supported. Enter in the server names, separated by commas, and the ! usernames and passwords separated by '\'s. The filter will also try ! and load a configuration file for each server (it will look in the ! same directory as the main configuration file, for a file called ! server.name.ini or .spambayes_server_name_rc), where you can change ! the folders to filter, the thresholds, or any other options (this ! currently does not include the database names). ! o AUTH CRAM-MD5 authentication is now supported, in addition to plaintext ! AUTH Login. ! o Many minor fixes and improvements. ! ! sb_pop3dnd.py ! ------------- ! Many improvements. Should now be useable. Binary now included. Tray ! app now included. ! o Spam classification and spam training have been separated out into ! separate folders. ! o No longer uses the web interface. Configuration is available via ! a dialog, training is via drag and drop between IMAP folders, and ! information is provided in the "Inbox" IMAP folder. ! ! New ! --- ! o Andrew Dalke's sb_culler.py script (an updated version) is now ! included in the source distribution. ! o The new hammie2cdb.py script converts standard spambayes.Storage ! databases into cdb databases usable by CdbClassifier. ! o The new postfixproxy.py script is a proxy filter for use with ! PostFix 2.1's content filter functionality. ! o The new pycksum.py script is a fuzzy checksum program. ! o The new showclues.py script outputs to stdout a 'show clues' report like ! the one that Outlook creates, with the clues, tokens and message stream. ! ! Other ! ----- ! o The sb_upload.py script's help string has been improved, and the ! -n (--null) command line option now works. The script is now also ! able to train as well as upload. ! o The tte.py script has had many improvements, including Python 2.3 ! compatibility, addition of a --ratio flag, deletion of correctly ! scoring messages, an improved scoring algorithm, and improved ! detection of whether a message is a miss or not. ! o A version of sb_bnfilter that is implemented in C (for speed) is ! now available. --- 68,105 ---- General ------- ! o Handling of errors when using the ZODB database backend has been ! improved. ! o The ZODB databases are now automatically packed, to save space. ! o The round accounting in tte.py has additional information. ! o Basic options are now available in sort+group.py ! o fpfn.py now works with unsures, and offers interactivity. Outlook Plugin -------------- ! o The folder dialogs were broken in 1.1a1; these have been fixed. ! POP3 Proxy (sb_server.py) ------------------------- ! o If an error occurs while filtering, an exception header is added to ! the message (instead of the SpamBayes headers). In 1.1a1, this ! exception header caused the rest of the message headers to move into ! the message body. This is now fixed. Web interface (sb_server.py and sb_imapfilter.py) ------------------------------------------------- ! o The default action for ham and spam in the review page is now 'discard'. ! We recommend using a 'train on false positives, false negatives and ! unsures' regime, and these are the most suitable defaults for that. ! o The "Notate To" and "Notate Subject" options should work correctly ! from the configuration page now. ! o The current row of the review page is now highlighted. ! IMAP Filter (sb_imapfilter.py) ------------------------------ ! o One-off errors in connecting to the server when using the -l option ! are handled better. ! o A problem with closing a folder twice was fixed. ! o Invalid dates are handled correctly. ! o Only selected folders are expunged. *************** *** 172,178 **** =================== The following bugs tracked via the Sourceforge system were fixed: ! 1078923, 1022848, 831864, 922063, 1051081, 903905, 1036601, 790757, ! 944109, 959937, 943852, 941639, 986353, 990700, 933473, 981970, 1071319, ! 1166146, 1175439 A URL containing the details of these bugs can be made by appending the --- 107,111 ---- =================== The following bugs tracked via the Sourceforge system were fixed: ! 1181160, 1179055, 1187208, 1182671, 1182754, 759917 A URL containing the details of these bugs can be made by appending the *************** *** 185,190 **** The following feature requests tracked via the Sourceforge system were added in this release: ! 870524, 940547, 715248, 938992, 1039057, 909088, 887984, 753708, 848365, ! 1144670, 1122067 A URL containing the details of these bugs can be made by appending the --- 118,122 ---- The following feature requests tracked via the Sourceforge system were added in this release: ! 1182703, 618932 A URL containing the details of these bugs can be made by appending the *************** *** 195,205 **** Patches integrated =================== ! The following patches tracked via the Sourceforge system were integrated for ! this release: ! 1052816, 962693, 831941, 858925, 800671, 1169939 ! ! A URL containing the details of these bugs can be made by appending the ! bug number to this URL: ! http://sourceforge.net/tracker/index.php?func=detail&group_id=61702&atid=498103&aid= --- 127,131 ---- Patches integrated =================== ! No patches tracked via the Sourceforge system were integrated for this release. From sjoerd at users.sourceforge.net Sun Nov 27 13:32:43 2005 From: sjoerd at users.sourceforge.net (Sjoerd Mullender) Date: Sun, 27 Nov 2005 13:32:43 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes message.py,1.70,1.71 Message-ID: <20051127123243.689081E400B@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv16807 Modified Files: message.py Log Message: SyntaxError: from __future__ imports must occur at the beginning of the file (although they can occor after a doc string). Index: message.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/message.py,v retrieving revision 1.70 retrieving revision 1.71 diff -C2 -d -r1.70 -r1.71 *** message.py 15 Nov 2005 00:35:57 -0000 1.70 --- message.py 27 Nov 2005 12:32:36 -0000 1.71 *************** *** 69,77 **** # Foundation license. __author__ = "Tim Stone " __credits__ = "Mark Hammond, Tony Meyer, all the spambayes contributors." - from __future__ import generators - try: True, False --- 69,77 ---- # Foundation license. + from __future__ import generators + __author__ = "Tim Stone " __credits__ = "Mark Hammond, Tony Meyer, all the spambayes contributors." try: True, False From sjoerd at users.sourceforge.net Sun Nov 27 13:40:16 2005 From: sjoerd at users.sourceforge.net (Sjoerd Mullender) Date: Sun, 27 Nov 2005 13:40:16 +0100 (CET) Subject: [Spambayes-checkins] spambayes/scripts sb_imapfilter.py,1.63,1.64 Message-ID: <20051127124016.1BFC61E400B@bag.python.org> Update of /cvsroot/spambayes/spambayes/scripts In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv18027 Modified Files: sb_imapfilter.py Log Message: BadIMAPResponseError has two arguments: command and response. Index: sb_imapfilter.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/scripts/sb_imapfilter.py,v retrieving revision 1.63 retrieving revision 1.64 diff -C2 -d -r1.63 -r1.64 *** sb_imapfilter.py 8 Jul 2005 03:24:40 -0000 1.63 --- sb_imapfilter.py 27 Nov 2005 12:40:07 -0000 1.64 *************** *** 313,318 **** # passed, we get a traceback, not just an 'invalid folder' # error, so raise our own error. ! raise BadIMAPResponseError("Cannot have empty string as " \ ! "folder name in select", "") # We *always* use SELECT and not EXAMINE, because this --- 313,319 ---- # passed, we get a traceback, not just an 'invalid folder' # error, so raise our own error. ! raise BadIMAPResponseError("select", ! "Cannot have empty string as " ! "folder name in select") # We *always* use SELECT and not EXAMINE, because this *************** *** 719,723 **** break else: ! raise BadIMAPResponseError("Cannot find saved message", "") # We need to update the UID, as it will have changed. --- 720,724 ---- break else: ! raise BadIMAPResponseError("recent", "Cannot find saved message") # We need to update the UID, as it will have changed. From anadelonbrin at users.sourceforge.net Sun Nov 27 23:05:57 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Sun, 27 Nov 2005 23:05:57 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes Options.py,1.130,1.131 Message-ID: <20051127220557.4ECE31E4036@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv3918/spambayes Modified Files: Options.py Log Message: Only default to ZODB if ZODB is importable. Index: Options.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v retrieving revision 1.130 retrieving revision 1.131 diff -C2 -d -r1.130 -r1.131 *** Options.py 27 Nov 2005 02:15:33 -0000 1.130 --- Options.py 27 Nov 2005 22:05:45 -0000 1.131 *************** *** 36,39 **** --- 36,58 ---- from OptionsClass import * + # A little magic. We'd like to use ZODB as the default storage, + # because we've had so many problems with bsddb, and we'd like to swap + # to new ZODB problems . However, apart from this, we only need + # a standard Python install - if the default was ZODB then we would + # need ZODB to be installed as well (which it will br for binary users, + # but might not be for source users). So what we do is check whether + # ZODB is importable and if it is, default to that, and if not, default + # to dbm. If ZODB is sometimes importable and sometimes not (e.g. you + # muck around with the PYTHONPATH), then this may not work well - the + # best idea would be to explicitly put the type in your configuration + # file. + try: + import ZODB + except ImportError: + DB_TYPE = "dbm", "hammie.db", "spambayes.messageinfo.db" + else: + del ZODB + DB_TYPE = "zodb", "hammie.fs", "messageinfo.fs" + # Format: # defaults is a dictionary, where the keys are the section names *************** *** 516,520 **** # loaded by the appropriate application only. "Storage" : ( ! ("persistent_use_database", _("Database backend"), "zodb", _("""SpamBayes can use either a ZODB or dbm database (quick to score one message) or a pickle (quick to train on huge amounts of messages). --- 535,539 ---- # loaded by the appropriate application only. "Storage" : ( ! ("persistent_use_database", _("Database backend"), DB_TYPE[0], _("""SpamBayes can use either a ZODB or dbm database (quick to score one message) or a pickle (quick to train on huge amounts of messages). *************** *** 523,527 **** ("zeo", "zodb", "cdb", "mysql", "pgsql", "dbm", "pickle"), RESTORE), ! ("persistent_storage_file", _("Storage file name"), "hammie.fs", _("""Spambayes builds a database of information that it gathers from incoming emails and from you, the user, to get better and --- 542,546 ---- ("zeo", "zodb", "cdb", "mysql", "pgsql", "dbm", "pickle"), RESTORE), ! ("persistent_storage_file", _("Storage file name"), DB_TYPE[1], _("""Spambayes builds a database of information that it gathers from incoming emails and from you, the user, to get better and *************** *** 532,536 **** FILE_WITH_PATH, DO_NOT_RESTORE), ! ("messageinfo_storage_file", _("Message information file name"), "messageinfo.fs", _("""Spambayes builds a database of information about messages that it has already seen and trained or classified. This --- 551,555 ---- FILE_WITH_PATH, DO_NOT_RESTORE), ! ("messageinfo_storage_file", _("Message information file name"), DB_TYPE[2], _("""Spambayes builds a database of information about messages that it has already seen and trained or classified. This *************** *** 1197,1201 **** } - # `optionsPathname` is the pathname of the last ini file in the list. # This is where the web-based configuration page will write its changes. --- 1216,1219 ---- From anadelonbrin at users.sourceforge.net Mon Nov 28 01:07:12 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon, 28 Nov 2005 01:07:12 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes Stats.py, 1.16, 1.17 message.py, 1.71, 1.72 Message-ID: <20051128000712.5D3611E4016@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv30121/spambayes Modified Files: Stats.py message.py Log Message: Get Stats to work with ZODB messageinfo db. Index: Stats.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/Stats.py,v retrieving revision 1.16 retrieving revision 1.17 diff -C2 -d -r1.16 -r1.17 *** Stats.py 6 Apr 2005 02:42:04 -0000 1.16 --- Stats.py 28 Nov 2005 00:07:03 -0000 1.17 *************** *** 127,131 **** self.ResetTotal() totals = self.totals ! for msg_id in self.messageinfo_db.db.keys(): # Skip the date key. if msg_id == STATS_START_KEY: --- 127,131 ---- self.ResetTotal() totals = self.totals ! for msg_id in self.messageinfo_db.keys(): # Skip the date key. if msg_id == STATS_START_KEY: Index: message.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/message.py,v retrieving revision 1.71 retrieving revision 1.72 diff -C2 -d -r1.71 -r1.72 *** message.py 27 Nov 2005 12:32:36 -0000 1.71 --- message.py 28 Nov 2005 00:07:03 -0000 1.72 *************** *** 199,202 **** --- 199,205 ---- self.store() + def keys(self): + return self.db.keys() + class MessageInfoPickle(MessageInfoBase): def __init__(self, db_name, pickle_type=1): *************** *** 283,286 **** --- 286,291 ---- # Override ZODBClassifier.__setattr__ object.__setattr__(self, att, value) + def keys(self): + return self.classifier.keys() From anadelonbrin at users.sourceforge.net Mon Nov 28 03:05:05 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon, 28 Nov 2005 03:05:05 +0100 (CET) Subject: [Spambayes-checkins] spambayes/scripts sb_server.py,1.47,1.48 Message-ID: <20051128020505.6E8531E400F@bag.python.org> Update of /cvsroot/spambayes/spambayes/scripts In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv22889/scripts Modified Files: sb_server.py Log Message: Close messageinfo database on close. Only close state if it was prepared. Index: sb_server.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/scripts/sb_server.py,v retrieving revision 1.47 retrieving revision 1.48 diff -C2 -d -r1.47 -r1.48 *** sb_server.py 26 Nov 2005 05:12:49 -0000 1.47 --- sb_server.py 28 Nov 2005 02:04:57 -0000 1.48 *************** *** 795,798 **** --- 795,803 ---- self.bayes.close() self.bayes = None + if self.mdb is not None: + self.mdb.store() + self.mdb.close() + self.mdb = None + spambayes.message.Message.reload_message_info_db() self.spamCorpus = self.hamCorpus = self.unknownCorpus = None *************** *** 996,1001 **** del proxyListeners[:] ! # Close the state (which saves if necessary) ! state.close() # And get a new one going. state = State() --- 1001,1007 ---- del proxyListeners[:] ! if state.prepared: ! # Close the state (which saves if necessary) ! state.close() # And get a new one going. state = State() From anadelonbrin at users.sourceforge.net Mon Nov 28 03:06:51 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon, 28 Nov 2005 03:06:51 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes storage.py,1.55,1.56 Message-ID: <20051128020651.9B03C1E400F@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv23134/spambayes Modified Files: storage.py Log Message: Don't pack in read-only mode. Don't use "db" as an attribute, as subclasses might want that. Close databases after converting. Handle converting empty databases. Open database to convert in read-only mode. Index: storage.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v retrieving revision 1.55 retrieving revision 1.56 diff -C2 -d -r1.55 -r1.56 *** storage.py 26 Nov 2005 06:44:15 -0000 1.55 --- storage.py 28 Nov 2005 02:06:42 -0000 1.56 *************** *** 729,734 **** self.create_storage() ! self.db = ZODB.DB(self.storage) ! self.conn = self.db.open() root = self.conn.root() --- 729,734 ---- self.create_storage() ! self.DB = ZODB.DB(self.storage) ! self.conn = self.DB.open() root = self.conn.root() *************** *** 791,795 **** # Do the closing. ! self.db.close() # We don't make any use of the 'undo' capabilities of the --- 791,795 ---- # Do the closing. ! self.DB.close() # We don't make any use of the 'undo' capabilities of the *************** *** 799,803 **** # XXX What is the 'referencesf' parameter for pack()? It doesn't # XXX seem to do anything according to the source. ! if hasattr(self.storage, "pack"): self.storage.pack(time.time()-60*60*24, None) self.storage.close() --- 799,803 ---- # XXX What is the 'referencesf' parameter for pack()? It doesn't # XXX seem to do anything according to the source. ! if self.mode != 'r' and hasattr(self.storage, "pack"): self.storage.pack(time.time()-60*60*24, None) self.storage.close() *************** *** 1016,1025 **** new_type = auto_type ! old_bayes = open_storage(old_name, old_type) new_bayes = open_storage(new_name, new_type) words = old_bayes._wordinfokeys() ! new_bayes.nham = old_bayes.nham ! new_bayes.nspam = old_bayes.nspam print >> sys.stderr, "Converting %s (%s database) to " \ --- 1016,1031 ---- new_type = auto_type ! old_bayes = open_storage(old_name, old_type, 'r') new_bayes = open_storage(new_name, new_type) words = old_bayes._wordinfokeys() ! try: ! new_bayes.nham = old_bayes.nham ! except AttributeError: ! new_bayes.nham = 0 ! try: ! new_bayes.nspam = old_bayes.nspam ! except AttributeError: ! new_bayes.nspam = 0 print >> sys.stderr, "Converting %s (%s database) to " \ *************** *** 1030,1037 **** for word in words: new_bayes._wordinfoset(word, old_bayes._wordinfoget(word)) ! print "Storing database, please be patient..." new_bayes.store() ! print "Conversion complete." def ensureDir(dirname): --- 1036,1045 ---- for word in words: new_bayes._wordinfoset(word, old_bayes._wordinfoget(word)) + old_bayes.close() ! print >> sys.stderr, "Storing database, please be patient..." new_bayes.store() ! print >> sys.stderr, "Conversion complete." ! new_bayes.close() def ensureDir(dirname): From anadelonbrin at users.sourceforge.net Mon Nov 28 03:08:10 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon, 28 Nov 2005 03:08:10 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes message.py,1.72,1.73 Message-ID: <20051128020810.5E08E1E400F@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv23239/spambayes Modified Files: message.py Log Message: Add ability for messageinfo class property to be reloaded. Fix the fix for stats to work. Index: message.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/message.py,v retrieving revision 1.72 retrieving revision 1.73 diff -C2 -d -r1.72 -r1.73 *** message.py 28 Nov 2005 00:07:03 -0000 1.72 --- message.py 28 Nov 2005 02:08:03 -0000 1.73 *************** *** 125,129 **** def __len__(self): ! return len(self.db) def get_statistics_start_date(self): --- 125,129 ---- def __len__(self): ! return len(self.keys()) def get_statistics_start_date(self): *************** *** 283,291 **** storage.ZODBClassifier.__init__(self, db_name) self.classifier.store = self.store def __setattr__(self, att, value): # Override ZODBClassifier.__setattr__ object.__setattr__(self, att, value) - def keys(self): - return self.classifier.keys() --- 283,290 ---- storage.ZODBClassifier.__init__(self, db_name) self.classifier.store = self.store + self.db = self.classifier def __setattr__(self, att, value): # Override ZODBClassifier.__setattr__ object.__setattr__(self, att, value) *************** *** 364,367 **** --- 363,369 ---- self._set_class_message_info_db(value) message_info_db = property(_get_message_info_db, _set_message_info_db) + def _reload_message_info_db(klass): + klass._message_info_db = None + reload_message_info_db = classmethod(_reload_message_info_db) # This function (and it's hackishness) can be avoided by using From anadelonbrin at users.sourceforge.net Mon Nov 28 03:10:22 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon, 28 Nov 2005 03:10:22 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes ProxyUI.py, 1.62, 1.63 UserInterface.py, 1.59, 1.60 Message-ID: <20051128021022.915791E400F@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv23526/spambayes Modified Files: ProxyUI.py UserInterface.py Log Message: If the user changes the database type in the configuration page, automatically convert the database for them, rather than crashing. Index: ProxyUI.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/ProxyUI.py,v retrieving revision 1.62 retrieving revision 1.63 diff -C2 -d -r1.62 -r1.63 *** ProxyUI.py 14 Apr 2005 06:18:16 -0000 1.62 --- ProxyUI.py 28 Nov 2005 02:10:14 -0000 1.63 *************** *** 7,11 **** This module implements a browser based Spambayes user interface for the ! POP3 proxy and SMTP proxy. Users may use it to interface with the proxies. --- 7,11 ---- This module implements a browser based Spambayes user interface for the ! POP3, IMAP4 SMTP proxies. Users may use it to interface with the proxies. *************** *** 88,91 **** --- 88,94 ---- ('pop3proxy', 'remote_servers'), ('pop3proxy', 'listen_ports'), + # ('IMAP4 Proxy Options', None), + # ('imap4proxy', 'remote_servers'), + # ('imap4proxy', 'listen_ports'), ('SMTP Proxy Options', None), ('smtpproxy', 'remote_servers'), *************** *** 169,174 **** proxy_state.stats) state = proxy_state self.state_recreator = state_recreator # ugly ! self.app_for_version = "SpamBayes POP3 Proxy" self.previous_sort = None if not proxy_state.can_stop: --- 172,178 ---- proxy_state.stats) state = proxy_state + self.state = proxy_state self.state_recreator = state_recreator # ugly ! self.app_for_version = "SpamBayes Proxy" self.previous_sort = None if not proxy_state.can_stop: *************** *** 228,232 **** of the given day, the end timestamp of the given day, and the formatted date of the given day.""" - # This probably works on Summertime-shift days; time will tell. 8-) this = time.localtime(timestamp) start = (this[0], this[1], this[2], 0, 0, 0, this[6], this[7], this[8]) --- 232,235 ---- *************** *** 777,780 **** --- 780,784 ---- state = self.state_recreator() self.classifier = state.bayes + self.state = state def verifyInput(self, parms, pmap): Index: UserInterface.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/UserInterface.py,v retrieving revision 1.59 retrieving revision 1.60 diff -C2 -d -r1.59 -r1.60 *** UserInterface.py 15 Nov 2005 00:13:54 -0000 1.59 --- UserInterface.py 28 Nov 2005 02:10:14 -0000 1.60 *************** *** 835,838 **** --- 835,841 ---- return + old_database_type = options["Storage", "persistent_use_database"] + old_name = options["Storage", "persistent_storage_file"] + for name, value in parms.items(): sect, opt = name.split('_', 1) *************** *** 847,850 **** --- 850,885 ---- options.update_file(optionsPathname) + + # If the database type changed, then convert it for them. + if options["Storage", "persistent_use_database"] != \ + old_database_type and os.path.exists(old_name): + new_name = options["Storage", "persistent_storage_file"] + new_type = options["Storage", "persistent_use_database"] + self.state.close() + try: + os.remove(new_name + ".tmp") + except OSError: + pass + storage.convert(old_name, old_database_type, + new_name + ".tmp", new_type) + if os.path.exists(new_name): + try: + os.remove(new_name + ".old") + except OSError: + pass + os.rename(new_name, new_name + ".old") + os.rename(new_name + ".tmp", new_name) + # Messageinfo db is not converted. + if os.path.exists(options["Storage", + "messageinfo_storage_file"]): + try: + os.remove(options["Storage", + "messageinfo_storage_file"] + ".old") + except OSError: + pass + os.rename(options["Storage", "messageinfo_storage_file"], + options["Storage", + "messageinfo_storage_file"] + ".old") + self.reReadOptions() From anadelonbrin at users.sourceforge.net Mon Nov 28 11:51:05 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon, 28 Nov 2005 11:51:05 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes Stats.py,1.17,1.18 Message-ID: <20051128105105.793C71E4272@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv15343/spambayes Modified Files: Stats.py Log Message: Update message creation. Index: Stats.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/Stats.py,v retrieving revision 1.17 retrieving revision 1.18 diff -C2 -d -r1.17 -r1.18 *** Stats.py 28 Nov 2005 00:07:03 -0000 1.17 --- Stats.py 28 Nov 2005 10:50:56 -0000 1.18 *************** *** 131,135 **** if msg_id == STATS_START_KEY: continue ! m = Message(msg_id, self.messageinfo_db) self.messageinfo_db.load_msg(m) --- 131,135 ---- if msg_id == STATS_START_KEY: continue ! m = Message(msg_id) self.messageinfo_db.load_msg(m) From anadelonbrin at users.sourceforge.net Mon Nov 28 11:51:53 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon, 28 Nov 2005 11:51:53 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes storage.py,1.56,1.57 Message-ID: <20051128105153.667E11E4079@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv15514/spambayes Modified Files: storage.py Log Message: Don't hide other dbmstorage errors. Index: storage.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v retrieving revision 1.56 retrieving revision 1.57 diff -C2 -d -r1.56 -r1.57 *** storage.py 28 Nov 2005 02:06:42 -0000 1.56 --- storage.py 28 Nov 2005 10:51:44 -0000 1.57 *************** *** 956,959 **** --- 956,960 ---- "such as bsddb (see http://sf.net/projects/pybsddb)." sys.exit() + raise # The different database types that are available. From anadelonbrin at users.sourceforge.net Mon Nov 28 11:52:37 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon, 28 Nov 2005 11:52:37 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes message.py,1.73,1.74 Message-ID: <20051128105237.497061E4079@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv15641/spambayes Modified Files: message.py Log Message: Remove unnecessary method. Index: message.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/message.py,v retrieving revision 1.73 retrieving revision 1.74 diff -C2 -d -r1.73 -r1.74 *** message.py 28 Nov 2005 02:08:03 -0000 1.73 --- message.py 28 Nov 2005 10:52:29 -0000 1.74 *************** *** 345,349 **** # it as a regular attribute, so need to make it a property. To make # a classmethod property, we have to jump through some hoops, which we ! # deserver for not doing it right in the first place. _message_info_db = None def _get_class_message_info_db(klass): --- 345,349 ---- # it as a regular attribute, so need to make it a property. To make # a classmethod property, we have to jump through some hoops, which we ! # deserve for not doing it right in the first place. _message_info_db = None def _get_class_message_info_db(klass): *************** *** 363,369 **** self._set_class_message_info_db(value) message_info_db = property(_get_message_info_db, _set_message_info_db) - def _reload_message_info_db(klass): - klass._message_info_db = None - reload_message_info_db = classmethod(_reload_message_info_db) # This function (and it's hackishness) can be avoided by using --- 363,366 ---- From anadelonbrin at users.sourceforge.net Mon Nov 28 11:53:20 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon, 28 Nov 2005 11:53:20 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes UserInterface.py, 1.60, 1.61 Message-ID: <20051128105320.E9B911E4079@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv15790/spambayes Modified Files: UserInterface.py Log Message: Have subclasses close the databases. Index: UserInterface.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/UserInterface.py,v retrieving revision 1.60 retrieving revision 1.61 diff -C2 -d -r1.60 -r1.61 *** UserInterface.py 28 Nov 2005 02:10:14 -0000 1.60 --- UserInterface.py 28 Nov 2005 10:53:12 -0000 1.61 *************** *** 856,860 **** new_name = options["Storage", "persistent_storage_file"] new_type = options["Storage", "persistent_use_database"] ! self.state.close() try: os.remove(new_name + ".tmp") --- 856,860 ---- new_name = options["Storage", "persistent_storage_file"] new_type = options["Storage", "persistent_use_database"] ! self.close_database() try: os.remove(new_name + ".tmp") From anadelonbrin at users.sourceforge.net Mon Nov 28 11:54:25 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon, 28 Nov 2005 11:54:25 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes ProxyUI.py,1.63,1.64 Message-ID: <20051128105425.EB5A71E4079@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv15964/spambayes Modified Files: ProxyUI.py Log Message: Close the database here rather than in parent class. Index: ProxyUI.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/ProxyUI.py,v retrieving revision 1.63 retrieving revision 1.64 diff -C2 -d -r1.63 -r1.64 *** ProxyUI.py 28 Nov 2005 02:10:14 -0000 1.63 --- ProxyUI.py 28 Nov 2005 10:54:18 -0000 1.64 *************** *** 172,176 **** proxy_state.stats) state = proxy_state - self.state = proxy_state self.state_recreator = state_recreator # ugly self.app_for_version = "SpamBayes Proxy" --- 172,175 ---- *************** *** 767,770 **** --- 766,772 ---- return messageInfo + def close_database(self): + state.close() + def reReadOptions(self): """Called by the config page when the user saves some new options, or *************** *** 780,784 **** state = self.state_recreator() self.classifier = state.bayes - self.state = state def verifyInput(self, parms, pmap): --- 782,785 ---- From anadelonbrin at users.sourceforge.net Mon Nov 28 11:55:15 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon, 28 Nov 2005 11:55:15 +0100 (CET) Subject: [Spambayes-checkins] spambayes/scripts sb_server.py,1.48,1.49 Message-ID: <20051128105515.2860D1E4079@bag.python.org> Update of /cvsroot/spambayes/spambayes/scripts In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv16163/scripts Modified Files: sb_server.py Log Message: Simplify reloading messageinfo db. Index: sb_server.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/scripts/sb_server.py,v retrieving revision 1.48 retrieving revision 1.49 diff -C2 -d -r1.48 -r1.49 *** sb_server.py 28 Nov 2005 02:04:57 -0000 1.48 --- sb_server.py 28 Nov 2005 10:55:07 -0000 1.49 *************** *** 799,803 **** self.mdb.close() self.mdb = None ! spambayes.message.Message.reload_message_info_db() self.spamCorpus = self.hamCorpus = self.unknownCorpus = None --- 799,803 ---- self.mdb.close() self.mdb = None ! spambayes.message.Message.message_info_db = None self.spamCorpus = self.hamCorpus = self.unknownCorpus = None From anadelonbrin at users.sourceforge.net Mon Nov 28 11:57:12 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon, 28 Nov 2005 11:57:12 +0100 (CET) Subject: [Spambayes-checkins] spambayes/spambayes ImapUI.py,1.47,1.48 Message-ID: <20051128105712.EB8FA1E4079@bag.python.org> Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv16550/spambayes Modified Files: ImapUI.py Log Message: Handle database changes while filter is running. Actually, there will still be problems if it is changing while the filter is actually running, rather than waiting to run next; fixing that would be somewhat more complicated. Index: ImapUI.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/ImapUI.py,v retrieving revision 1.47 retrieving revision 1.48 diff -C2 -d -r1.47 -r1.48 *** ImapUI.py 13 May 2005 04:24:27 -0000 1.47 --- ImapUI.py 28 Nov 2005 10:57:07 -0000 1.48 *************** *** 13,22 **** The following functions are currently included: [From the base class UserInterface] ! onClassify - classify a given message ! onWordquery - query a word from the database ! onTrain - train a message or mbox ! onSave - save the database and possibly shutdown [Here] ! onHome - a home page with various options To do: --- 13,22 ---- The following functions are currently included: [From the base class UserInterface] ! onClassify - classify a given message ! onWordquery - query a word from the database ! onTrain - train a message or mbox ! onSave - save the database and possibly shutdown [Here] ! onHome - a home page with various options To do: *************** *** 125,129 **** """Serves the HTML user interface for the proxies.""" def __init__(self, cls, imaps, pwds, imap_session_class, ! lang_manager=None, stats=None): global parm_map # Only offer SSL if it is available --- 125,130 ---- """Serves the HTML user interface for the proxies.""" def __init__(self, cls, imaps, pwds, imap_session_class, ! lang_manager=None, stats=None, ! close_db=None, change_db=None): global parm_map # Only offer SSL if it is available *************** *** 143,146 **** --- 144,149 ---- self.app_for_version = "SpamBayes IMAP Filter" self.imap_session_class = imap_session_class + self.close_database = close_db + self.change_db = change_db def onHome(self): *************** *** 175,183 **** restores the defaults.""" # Re-read the options. - self.classifier.store() import Options Options.load_options() global options from Options import options def onSave(self, how): --- 178,186 ---- restores the defaults.""" # Re-read the options. import Options Options.load_options() global options from Options import options + self.change_db() def onSave(self, how): From anadelonbrin at users.sourceforge.net Mon Nov 28 11:57:13 2005 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon, 28 Nov 2005 11:57:13 +0100 (CET) Subject: [Spambayes-checkins] spambayes/scripts sb_imapfilter.py,1.64,1.65 Message-ID: <20051128105713.033A31E407F@bag.python.org> Update of /cvsroot/spambayes/spambayes/scripts In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv16550/scripts Modified Files: sb_imapfilter.py Log Message: Handle database changes while filter is running. Actually, there will still be problems if it is changing while the filter is actually running, rather than waiting to run next; fixing that would be somewhat more complicated. Index: sb_imapfilter.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/scripts/sb_imapfilter.py,v retrieving revision 1.64 retrieving revision 1.65 diff -C2 -d -r1.64 -r1.65 *** sb_imapfilter.py 27 Nov 2005 12:40:07 -0000 1.64 --- sb_imapfilter.py 28 Nov 2005 10:57:07 -0000 1.65 *************** *** 146,150 **** from spambayes import message from spambayes.Options import options, get_pathname_option, optionsPathname ! from spambayes import tokenizer, storage, message, Dibbler from spambayes.UserInterface import UserInterfaceServer from spambayes.ImapUI import IMAPUserInterface, LoginFailure --- 146,150 ---- from spambayes import message from spambayes.Options import options, get_pathname_option, optionsPathname ! from spambayes import tokenizer, storage, Dibbler from spambayes.UserInterface import UserInterfaceServer from spambayes.ImapUI import IMAPUserInterface, LoginFailure *************** *** 1185,1191 **** imaps.append(IMAPSession(server, port, imapDebug, doExpunge)) httpServer = UserInterfaceServer(options["html_ui", "port"]) httpServer.register(IMAPUserInterface(classifier, imaps, pwds, ! IMAPSession, stats=stats)) launchBrowser=launchUI or options["html_ui", "launch_browser"] if sleepTime: --- 1185,1207 ---- imaps.append(IMAPSession(server, port, imapDebug, doExpunge)) + def close_db(): + message_db.store() + message_db.close() + message.Message().message_info_db.store() + message.Message().message_info_db.close() + message.Message.message_info_db = None + classifier.store() + classifier.close() + + def change_db(): + classifier = storage.open_storage(*storage.database_type(opts)) + message_db = message.open_storage(*message.database_type()) + imap_filter = IMAPFilter(classifier, message_db) + httpServer = UserInterfaceServer(options["html_ui", "port"]) httpServer.register(IMAPUserInterface(classifier, imaps, pwds, ! IMAPSession, stats=stats, ! close_db=close_db, ! change_db=change_db)) launchBrowser=launchUI or options["html_ui", "launch_browser"] if sleepTime: