From mhammond at users.sourceforge.net Mon Sep 1 00:04:10 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Mon Sep 1 02:04:18 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000/sandbox dump_props.py, 1.9, 1.10 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000/sandbox In directory sc8-pr-cvs1:/tmp/cvs-serv9969 Modified Files: dump_props.py Log Message: Extract the HTML from the RTF, just like the plugin. Index: dump_props.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/sandbox/dump_props.py,v retrieving revision 1.9 retrieving revision 1.10 diff -C2 -d -r1.9 -r1.10 *** dump_props.py 29 Jul 2003 00:35:24 -0000 1.9 --- dump_props.py 1 Sep 2003 06:04:08 -0000 1.10 *************** *** 47,50 **** --- 47,58 ---- all_props.sort() # sort by first tuple item, which is name :) for prop_name, prop_tag, prop_val in all_props: + # Do some magic rtf conversion + if PROP_ID(prop_tag) == PROP_ID(PR_RTF_COMPRESSED): + rtf_stream = item.OpenProperty(PR_RTF_COMPRESSED, pythoncom.IID_IStream, + 0, 0) + html_stream = mapi.WrapCompressedRTFStream(rtf_stream, 0) + prop_val = mapi.RTFStreamToHTML(html_stream) + prop_name = "PR_RTF_COMPRESSED (to HTML)" + prop_tag = PROP_TAG(PT_STRING8, PR_RTF_COMPRESSED) if get_large_props and \ PROP_TYPE(prop_tag)==PT_ERROR and \ From mhammond at users.sourceforge.net Mon Sep 1 00:04:37 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Mon Sep 1 02:04:40 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000/sandbox extract_prop.py, NONE, 1.1 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000/sandbox In directory sc8-pr-cvs1:/tmp/cvs-serv10074 Added Files: extract_prop.py Log Message: Tool to extract a binary property to a file. --- NEW FILE: extract_prop.py --- # Extract a property from a message to a file. This is about the only # way to extract huge properties such that the original data is available. import sys, os import mapi_driver from win32com.mapi import mapitags, mapi import pythoncom def DumpItemProp(item, prop, outfile): if type(prop)!=type(0): # see if a mapitags contant try: prop = mapitags.__dict__[prop] except KeyError: # resolve as a name props = ( (mapi.PS_PUBLIC_STRINGS, prop), ) propIds = obj.GetIDsFromNames(props, 0) prop = mapitags.PROP_TAG( mapitags.PT_UNSPECIFIED, mapitags.PROP_ID(propIds[0])) hr, data = item.GetProps((prop,), 0) prop_tag, prop_val = data[0] # Do some magic rtf conversion if mapitags.PROP_ID(prop_tag) == mapitags.PROP_ID(mapitags.PR_RTF_COMPRESSED): rtf_stream = item.OpenProperty(mapitags.PR_RTF_COMPRESSED, pythoncom.IID_IStream, 0, 0) html_stream = mapi.WrapCompressedRTFStream(rtf_stream, 0) chunks = [] while 1: chunk = html_stream.Read(4096) if not chunk: break chunks.append(chunk) prop_val = "".join(chunks) elif mapitags.PROP_TYPE(prop_tag)==mapitags.PT_ERROR and \ prop_val in [mapi.MAPI_E_NOT_ENOUGH_MEMORY,'MAPI_E_NOT_ENOUGH_MEMORY']: prop_tag = mapitags.PROP_TAG(mapitags.PT_BINARY, mapitags.PROP_ID(prop_tag)) stream = item.OpenProperty(prop_tag, pythoncom.IID_IStream, 0, 0) chunks = [] while 1: chunk = stream.Read(4096) if not chunk: break chunks.append(chunk) prop_val = "".join(chunks) outfile.write(prop_val) def DumpProp(driver, mapi_folder, subject, prop_tag, outfile): hr, data = mapi_folder.GetProps( (mapitags.PR_DISPLAY_NAME_A,), 0) name = data[0][1] items = driver.GetItemsWithValue(mapi_folder, mapitags.PR_SUBJECT_A, subject) num = 0 for item in items: if num > 1: print >> sys.stderr, "Warning: More than one matching item - ignoring" break DumpItemProp(item, prop_tag, outfile) num += 1 if num==0: print >> sys.stderr, "Error: No matching items" def usage(driver): folder_doc = driver.GetFolderNameDoc() msg = """\ Usage: %s [-f foldername] [-o output_file] -p property_name subject of the message -f - Search for the message in the specified folder (default = Inbox) -p - Name of the property to dump -o - Output file to be created - default - stdout. Dumps all properties for all messages that match the subject. Subject matching is substring and ignore-case. %s Use the -n option to see all top-level folder names from all stores.""" \ % (os.path.basename(sys.argv[0]),folder_doc) print msg def main(): driver = mapi_driver.MAPIDriver() import getopt try: opts, args = getopt.getopt(sys.argv[1:], "np:f:o:") except getopt.error, e: print e print usage(driver) sys.exit(1) folder_name = prop_name = output_name = "" for opt, opt_val in opts: if opt == "-p": prop_name = opt_val elif opt == "-f": folder_name = opt_val elif opt == '-o': output_name = os.path.abspath(opt_val) elif opt == "-n": driver.DumpTopLevelFolders() sys.exit(1) else: print "Invalid arg" return if not folder_name: folder_name = "Inbox" # Assume this exists! subject = " ".join(args) if not subject: print "You must specify a subject" print usage(driver) sys.exit(1) if not prop_name: print "You must specify a property" print usage(driver) sys.exit(1) if output_name: output_file = file(output_name, "wb") else: output_file = sys.stdout try: folder = driver.FindFolder(folder_name) except ValueError, details: print details sys.exit(1) DumpProp(driver, folder, subject, prop_name, output_file) if __name__=='__main__': main() From richiehindle at users.sourceforge.net Mon Sep 1 00:07:29 2003 From: richiehindle at users.sourceforge.net (Richie Hindle) Date: Mon Sep 1 02:07:33 2003 Subject: [Spambayes-checkins] spambayes/spambayes Dibbler.py, 1.5, 1.6 Options.py, 1.68, 1.69 ProxyUI.py, 1.19, 1.20 UserInterface.py, 1.20, 1.21 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv10484/spambayes Modified Files: Dibbler.py Options.py ProxyUI.py UserInterface.py Log Message: HTTP-Auth support for the web interface (many thanks to Romain Guy). Index: Dibbler.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/Dibbler.py,v retrieving revision 1.5 retrieving revision 1.6 diff -C2 -d -r1.5 -r1.6 *** Dibbler.py 31 Aug 2003 02:09:40 -0000 1.5 --- Dibbler.py 1 Sep 2003 06:07:27 -0000 1.6 *************** *** 170,174 **** import StringIO ! import os, sys, re, time, traceback import socket, asyncore, asynchat, cgi, urlparse, webbrowser --- 170,174 ---- import StringIO ! import os, sys, re, time, traceback, md5, base64 import socket, asyncore, asynchat, cgi, urlparse, webbrowser *************** *** 291,294 **** --- 291,298 ---- """ + NO_AUTHENTICATION = "None" + BASIC_AUTHENTICATION = "Basic" + DIGEST_AUTHENTICATION = "Digest" + def __init__(self, port=('', 80), context=_defaultContext): """Create an `HTTPServer` for the given port.""" *************** *** 307,310 **** --- 311,338 ---- self._plugins.append(plugin) + def requestAuthenticationMode(self): + """Override: HTTP Authentication. It should return a value among + NO_AUTHENTICATION, BASIC_AUTHENTICATION and DIGEST_AUTHENTICATION. + The two last values will force HTTP authentication respectively + through Base64 and MD5 encodings.""" + return self.NO_AUTHENTICATION + + def isValidUser(self, name, password): + """Override: Return True for authorized logins.""" + return True + + def getPasswordForUser(self, name): + """Override: Return the password associated to the specified user + name.""" + return '' + + def getRealm(self): + """Override: Specify the HTTP authentication realm.""" + return "Dibbler application server" + + def getCancelMessage(self): + """Override: Specify the cancel message for an HTTP Authentication.""" + return "You must log in.""" + class _HTTPHandler(BrighterAsyncChat): *************** *** 385,388 **** --- 413,442 ---- params[name] = value[0] + # Parse the headers. + headersRegex = re.compile('([^:]*):\s*(.*)') + headersDict = dict([headersRegex.match(line).groups(2) + for line in headers.split('\r\n') + if headersRegex.match(line)]) + + # HTTP Basic/Digest Authentication support. + serverAuthMode = self._server.requestAuthenticationMode() + if serverAuthMode != HTTPServer.NO_AUTHENTICATION: + # The server wants us to authenticate the user. + authResult = False + authHeader = headersDict.get('Authorization') + if authHeader: + authMatch = re.search('(\w+)\s+(.*)', authHeader) + authenticationMode, login = authMatch.groups() + + if authenticationMode == HTTPServer.BASIC_AUTHENTICATION: + authResult = self._basicAuthentication(login) + elif authenticationMode == HTTPServer.DIGEST_AUTHENTICATION: + authResult = self._digestAuthentication(login, method) + else: + print >>sys.stdout, "Unknown mode: %s" % authenticationMode + + if not authResult: + self.writeUnauthorizedAccess(serverAuthMode) + # Find and call the methlet. '/eggs.gif' becomes 'onEggsGif'. if path == '/': *************** *** 492,495 **** --- 546,637 ---- content = '' self.push('\r\n'.join(headers) + str(content)) + + def writeUnauthorizedAccess(self, authenticationMode): + """Access is protected by HTTP authentication.""" + if authenticationMode == HTTPServer.BASIC_AUTHENTICATION: + authString = self._getBasicAuthString() + elif authenticationMode == HTTPServer.DIGEST_AUTHENTICATION: + authString = self._getDigestAuthString() + else: + self.writeError(500, "Inconsistent authentication mode.") + return + + headers = [] + headers.append('HTTP/1.0 401 Unauthorized') + headers.append('WWW-Authenticate: ' + authString) + headers.append('Connection: close') + headers.append('Content-Type: text/html') + headers.append('') + headers.append('') + self.write('\r\n'.join(headers) + self._server.getCancelMessage()) + self.close_when_done() + + def _getDigestAuthString(self): + """Builds the WWW-Authenticate header for Digest authentication.""" + authString = 'Digest realm="' + self._server.getRealm() + '"' + authString += ', nonce="' + self._getCurrentNonce() + '"' + authString += ', opaque="0000000000000000"' + authString += ', stale="false"' + authString += ', algorithm="MD5"' + authString += ', qop="auth"' + return authString + + def _getBasicAuthString(self): + """Builds the WWW-Authenticate header for Basic authentication.""" + return 'Basic realm="' + self._server.getRealm() + '"' + + def _getCurrentNonce(self): + """Returns the current nonce value. This value is a Base64 encoding + of current time plus one minute. This means the nonce will expire a + minute from now.""" + timeString = time.asctime(time.localtime(time.time() + 60)) + return base64.encodestring(timeString).rstrip('\n=') + + def _isValidNonce(self, nonce): + """Check if the specified nonce is still valid. A nonce is invalid + when its time converted value is lower than current time.""" + padAmount = len(nonce) % 4 + if padAmount > 0: padAmount = 4 - padAmount + nonce += '=' * (len(nonce) + padAmount) + + decoded = base64.decodestring(nonce) + return time.time() < time.mktime(time.strptime(decoded)) + + def _basicAuthentication(self, login): + """Performs a Basic HTTP authentication. Returns True when the user + has logged in successfully, False otherwise.""" + userName, password = base64.decodestring(login).split(':') + return self._server.isValidUser(userName, password) + + def _digestAuthentication(self, login, method): + """Performs a Digest HTTP authentication. Returns True when the user + has logged in successfully, False otherwise.""" + def stripQuotes(s): + return (s[0] == '"' and s[-1] == '"') and s[1:-1] or s + + options = dict([s.split('=') for s in login.split(", ")]) + userName = stripQuotes(options["username"]) + password = self._server.getPasswordForUser(userName) + nonce = stripQuotes(options["nonce"]) + + # The following computations are based upon RFC 2617. + A1 = "%s:%s:%s" % (userName, self._server.getRealm(), password) + HA1 = md5.new(A1).hexdigest() + A2 = "%s:%s" % (method, stripQuotes(options["uri"])) + HA2 = md5.new(A2).hexdigest() + + unhashedDigest = "" + if options.has_key("qop"): + unhashedDigest = "%s:%s:%s:%s:%s:%s" % \ + (HA1, nonce, + stripQuotes(options["nc"]), + stripQuotes(options["cnonce"]), + stripQuotes(options["qop"]), HA2) + else: + unhashedDigest = "%s:%s:%s" % (HA1, nonce, HA2) + hashedDigest = md5.new(unhashedDigest).hexdigest() + + return (stripQuotes(options["response"]) == hashedDigest and + self._isValidNonce(nonce)) Index: Options.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v retrieving revision 1.68 retrieving revision 1.69 diff -C2 -d -r1.68 -r1.69 *** Options.py 28 Aug 2003 21:15:06 -0000 1.68 --- Options.py 1 Sep 2003 06:07:27 -0000 1.69 *************** *** 832,835 **** --- 832,853 ---- want to quickly identify mail received via a mailing list.""", BOOLEAN, RESTORE), + + ("http_authentication", "HTTP Authentication", "None", + """This option lets you choose the security level of the web interface. + When selecting Basic or Digest, the user will be prompted a login and a + password to access the web interface. The Basic option is faster, but + transmits the password in clear on the network. The Digest option + encrypts the password before transmission.""", + ("None", "Basic", "Digest"), RESTORE), + + ("http_user_name", "User name", "admin", + """If you activated the HTTP authentication option, you can modify the + authorized user name here.""", + r"[\w]+", RESTORE), + + ("http_password", "Password", "admin", + """If you activated the HTTP authentication option, you can modify the + authorized user password here.""", + r"[\w]+", RESTORE), ), Index: ProxyUI.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/ProxyUI.py,v retrieving revision 1.19 retrieving revision 1.20 diff -C2 -d -r1.19 -r1.20 *** ProxyUI.py 26 Aug 2003 04:30:41 -0000 1.19 --- ProxyUI.py 1 Sep 2003 06:07:27 -0000 1.20 *************** *** 85,88 **** --- 85,91 ---- ('html_ui', 'display_to'), ('html_ui', 'allow_remote_connections'), + ('html_ui', 'http_authentication'), + ('html_ui', 'http_user_name'), + ('html_ui', 'http_password'), ('Header Options', None), ('pop3proxy', 'notate_to'), Index: UserInterface.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/UserInterface.py,v retrieving revision 1.20 retrieving revision 1.21 diff -C2 -d -r1.20 -r1.21 *** UserInterface.py 30 Aug 2003 21:35:16 -0000 1.20 --- UserInterface.py 1 Sep 2003 06:07:27 -0000 1.21 *************** *** 95,98 **** --- 95,115 ---- print 'User interface url is http://localhost:%d/' % (uiPort) + def requestAuthenticationMode(self): + return options["html_ui", "http_authentication"] + + def getRealm(self): + return "SpamBayes Web Interface" + + def isValidUser(self, name, password): + return (name == options["html_ui", "http_user_name"] and + password == options["html_ui", "http_password"]) + + def getPasswordForUser(self, name): + # There is only one login available in the web interface. + return options["html_ui", "http_password"] + + def getCancelMessage(self): + return "You must login to use SpamBayes.""" + class BaseUserInterface(Dibbler.HTTPPlugin): From mhammond at users.sourceforge.net Mon Sep 1 01:27:33 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Mon Sep 1 03:27:42 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000 about.html,1.21,1.22 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000 In directory sc8-pr-cvs1:/tmp/cvs-serv21197 Modified Files: about.html Log Message: Worm around Mozilla bug. Add full-stop and reword bit about Outlook rules. Index: about.html =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/about.html,v retrieving revision 1.21 retrieving revision 1.22 diff -C2 -d -r1.21 -r1.22 *** about.html 29 Aug 2003 15:18:22 -0000 1.21 --- about.html 1 Sep 2003 07:27:31 -0000 1.22 *************** *** 6,13 **** ! ! !
Logo 

SpamBayes Outlook Plugin

--- 6,19 ---- ! ! ! ! ! ! !
Logo  

SpamBayes Outlook Plugin

*************** *** 26,30 ****

Other links of interest

The main SpamBayes project page.
! The online SpamBayes FAQ
How to make a donation.
--- 32,36 ----

Other links of interest

The main SpamBayes project page.
! The online SpamBayes FAQ.
How to make a donation.
*************** *** 35,41 **** When Outlook is under load, SpamBayes may occasionally miss some messages (as Outlook doesn't tell us they appeared).  Also, if you ! use builtin Outlook rules, you may find that occasionally they fails to ! filter messages they normally would (as both SpamBayes and the builtin ! rules "fight" over processing the message).  Both of these problems can generally be solved by enabling background filtering of messages, via the SpamBayes Manager's Advanced --- 41,47 ---- When Outlook is under load, SpamBayes may occasionally miss some messages (as Outlook doesn't tell us they appeared).  Also, if you ! use builtin Outlook rules, you may find that occasionally they also
! fail to filter messages (as both SpamBayes and the builtin rules ! "fight" over processing the message).  Both of these problems can generally be solved by enabling background filtering of messages, via the SpamBayes Manager's Advanced From mhammond at users.sourceforge.net Mon Sep 1 01:29:50 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Mon Sep 1 03:29:54 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000 msgstore.py,1.70,1.71 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000 In directory sc8-pr-cvs1:/tmp/cvs-serv21408 Modified Files: msgstore.py Log Message: Handle MAPI failures fetching the message when trying to get a field value. New RTF->HTML code - however, it won't actually extract HTML until you upgrade your win32all to a version that is yet to be released. Index: msgstore.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/msgstore.py,v retrieving revision 1.70 retrieving revision 1.71 diff -C2 -d -r1.70 -r1.71 *** msgstore.py 31 Aug 2003 05:38:52 -0000 1.70 --- msgstore.py 1 Sep 2003 07:29:48 -0000 1.71 *************** *** 424,427 **** --- 424,454 ---- return ret + # Some nasty stuff for getting RTF out of the message + _have_complained_about_missing_rtf = False + def GetHTMLFromRTFProperty(mapi_object, prop_tag = PR_RTF_COMPRESSED): + global _have_complained_about_missing_rtf + try: + rtf_stream = mapi_object.OpenProperty(prop_tag, pythoncom.IID_IStream, + 0, 0) + except pythoncom.com_error, details: + if not IsNotFoundCOMException(details): + print "ERROR getting RTF body", details + return "" + try: + html_stream = mapi.WrapCompressedRTFStream(rtf_stream, 0) + except AttributeError: + if not _have_complained_about_missing_rtf: + print "*" * 50 + print "Sorry, but you need to update to a new win32all (158 or " + print "later), so we correctly get the HTML from messages." + print "See http://starship.python.net/crew/mhammond/win32" + print "*" * 50 + _have_complained_about_missing_rtf = True + return "" + html = mapi.RTFStreamToHTML(html_stream) + # html may be None if not RTF originally from HTML, but here we + # always want a string + return html or '' + class MAPIMsgStoreFolder(MsgStoreMsg): def __init__(self, msgstore, id, name, count): *************** *** 705,709 **** def _GetMessageTextParts(self): ! # This is finally reliable. The only messages this now fails for # are for "forwarded" messages, where the forwards are actually # in an attachment. Later. --- 732,736 ---- def _GetMessageTextParts(self): ! # This is almost reliable :). The only messages this now fails for # are for "forwarded" messages, where the forwards are actually # in an attachment. Later. *************** *** 720,723 **** --- 747,753 ---- html = self._GetPotentiallyLargeStringProp(prop_ids[1], data[1]) headers = self._GetPotentiallyLargeStringProp(prop_ids[2], data[2]) + # xxx - not sure what to do if we have both. + if not html: + html = GetHTMLFromRTFProperty(self.mapi_object) # Some Outlooks deliver a strange notion of headers, including *************** *** 937,941 **** def GetField(self, prop, raise_errors = False): ! self._EnsureObject() if type(prop) != type(0): props = ( (mapi.PS_PUBLIC_STRINGS, prop), ) --- 967,977 ---- def GetField(self, prop, raise_errors = False): ! try: ! self._EnsureObject() ! except pythoncom.com_error, details: ! if not IsNotFoundCOMException(details): ! print "ERROR: Could not open an object to fetch a field" ! print details ! return None if type(prop) != type(0): props = ( (mapi.PS_PUBLIC_STRINGS, prop), ) From anadelonbrin at users.sourceforge.net Mon Sep 1 03:05:54 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon Sep 1 05:06:02 2003 Subject: [Spambayes-checkins] spambayes/spambayes UserInterface.py, 1.21, 1.22 classifier.py, 1.6, 1.7 storage.py, 1.24, 1.25 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv4810/spambayes Modified Files: UserInterface.py classifier.py storage.py Log Message: Add the ability to get a list of the words (i.e. keys) in a classifier (SQL versions still coming...) Add [ 796832 ] Word query should show all words starting with certain text (which essentially is the ability to do a wildcard search through the classifier database through the web interface). Index: UserInterface.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/UserInterface.py,v retrieving revision 1.21 retrieving revision 1.22 diff -C2 -d -r1.21 -r1.22 *** UserInterface.py 1 Sep 2003 06:07:27 -0000 1.21 --- UserInterface.py 1 Sep 2003 09:05:51 -0000 1.22 *************** *** 281,304 **** def onWordquery(self, word): if word == "": stats = "You must enter a word." else: word = word.lower() ! wordinfo = classifier._wordinfoget(word) ! if wordinfo: ! stats = self.html.wordStats.clone() ! stats.spamcount = wordinfo.spamcount ! stats.hamcount = wordinfo.hamcount ! stats.spamprob = classifier.probability(wordinfo) else: ! stats = "%r does not exist in the database." % cgi.escape(word) query = self.html.wordQuery.clone() ! query.word.value = word ! statsBox = self._buildBox("Statistics for %r" % cgi.escape(word), ! 'status.gif', stats) queryBox = self._buildBox("Word query", 'query.gif', query) self._writePreamble("Word query") ! self.write(statsBox + queryBox) self._writePostamble() --- 281,356 ---- def onWordquery(self, word): + wildcard_limit = 10 + statsBoxes = [] if word == "": stats = "You must enter a word." + statsBoxes.append(self._buildBox("Statistics for %r" % \ + cgi.escape(word), + 'status.gif', stats)) else: word = word.lower() ! if word[-1] == '*': ! # Wildcard search - list all words that start with word[:-1] ! word = word[:-1] ! reached_limit = False ! for w in classifier._wordinfokeys(): ! if not reached_limit and len(statsBoxes) > wildcard_limit: ! reached_limit = True ! over_limit = 0 ! if w.startswith(word): ! if reached_limit: ! over_limit += 1 ! else: ! wordinfo = classifier._wordinfoget(w) ! stats = self.html.wordStats.clone() ! stats.spamcount = wordinfo.spamcount ! stats.hamcount = wordinfo.hamcount ! stats.spamprob = classifier.probability(wordinfo) ! box = self._buildBox("Statistics for %r" % \ ! cgi.escape(w), ! 'status.gif', stats) ! statsBoxes.append(box) ! if len(statsBoxes) == 0: ! stats = "There are no words that begin with '%s' " \ ! "in the database." % (word,) ! # We build a box for each word; I'm not sure this is ! # produces the nicest results, but it's ok with a ! # limited number of words. ! statsBoxes.append(self._buildBox("Statistics for %s" % \ ! cgi.escape(word), ! 'status.gif', stats)) ! elif reached_limit: ! if over_limit == 1: ! singles = ["was", "match", "is"] ! else: ! singles = ["were", "matches", "are"] ! stats = "There %s %d additional %s that %s not " \ ! "shown here." % (singles[0], over_limit, ! singles[1], singles[2]) ! box = self._buildBox("Statistics for '%s*'" % \ ! cgi.escape(word), 'status.gif', ! stats) ! statsBoxes.append(box) else: ! # Optimised version for non-wildcard searches ! wordinfo = classifier._wordinfoget(word) ! if wordinfo: ! stats = self.html.wordStats.clone() ! stats.spamcount = wordinfo.spamcount ! stats.hamcount = wordinfo.hamcount ! stats.spamprob = classifier.probability(wordinfo) ! else: ! stats = "%r does not exist in the database." % cgi.escape(word) ! statsBoxes.append(self._buildBox("Statistics for %r" % \ ! cgi.escape(word), ! 'status.gif', stats)) query = self.html.wordQuery.clone() ! query.word.value = "%s" % (word,) queryBox = self._buildBox("Word query", 'query.gif', query) self._writePreamble("Word query") ! for box in statsBoxes: ! self.write(box) ! self.write(queryBox) self._writePostamble() Index: classifier.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/classifier.py,v retrieving revision 1.6 retrieving revision 1.7 diff -C2 -d -r1.6 -r1.7 *** classifier.py 26 May 2003 04:55:40 -0000 1.6 --- classifier.py 1 Sep 2003 09:05:51 -0000 1.7 *************** *** 469,472 **** --- 469,475 ---- del self.wordinfo[word] + def _wordinfokeys(self): + return self.wordinfo.keys() + Bayes = Classifier Index: storage.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v retrieving revision 1.24 retrieving revision 1.25 diff -C2 -d -r1.24 -r1.25 *** storage.py 31 Aug 2003 21:10:58 -0000 1.24 --- storage.py 1 Sep 2003 09:05:51 -0000 1.25 *************** *** 269,272 **** --- 269,277 ---- self.changed_words[word] = WORD_DELETED + def _wordinfokeys(self): + wordinfokeys = self.db.keys() + del wordinfokeys[wordinfokeys.index(self.statekey)] + return wordinfokeys + class SQLClassifier(classifier.Classifier): From anadelonbrin at users.sourceforge.net Mon Sep 1 03:16:11 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon Sep 1 05:16:16 2003 Subject: [Spambayes-checkins] spambayes CHANGELOG.txt, 1.14, 1.15 WHAT_IS_NEW.txt, 1.11, 1.12 Message-ID: Update of /cvsroot/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv6042 Modified Files: CHANGELOG.txt WHAT_IS_NEW.txt Log Message: Bring up to date. Index: CHANGELOG.txt =================================================================== RCS file: /cvsroot/spambayes/spambayes/CHANGELOG.txt,v retrieving revision 1.14 retrieving revision 1.15 diff -C2 -d -r1.14 -r1.15 *** CHANGELOG.txt 1 Sep 2003 05:47:04 -0000 1.14 --- CHANGELOG.txt 1 Sep 2003 09:16:08 -0000 1.15 *************** *** 1,4 **** --- 1,6 ---- Alpha Release 5 =============== + Richie Hindle 01/09/2003 Integrated [ 791393 ] HTTP Authentication, which closes #791319. + Tony Meyer 01/09/2003 Added [ 796832 ] Word query should show all words starting with certain text Tony Meyer 01/09/2003 Fix for [ 797316 ] Extra CRLF to smtp server causes garbage error. Richie Hindle 31/08/2003 The web UI's Shutdown command, and stopping the pop3proxy_service, now wait for any open proxy connections to finish before exiting the process. Index: WHAT_IS_NEW.txt =================================================================== RCS file: /cvsroot/spambayes/spambayes/WHAT_IS_NEW.txt,v retrieving revision 1.11 retrieving revision 1.12 diff -C2 -d -r1.11 -r1.12 *** WHAT_IS_NEW.txt 1 Sep 2003 05:47:04 -0000 1.11 --- WHAT_IS_NEW.txt 1 Sep 2003 09:16:08 -0000 1.12 *************** *** 93,96 **** --- 93,101 ---- o Prevent the "Show clues" links on the web interface's training page from word-wrapping and making all the table rows two lines high. + o You can now put "*" at the end of a word in the "Word Query" box + on the web interface, and have it show you the first ten words, + and how many words there are in total, in the database that start with + that word. + o The web interface now supports HTTP-Auth. o Added a new script (code-named 'overkill.py') which enables 'drag and drop' training for POP3 users. This is currently still in *************** *** 155,159 **** The following feature requests tracked via the Sourceforge system were added: ! 789916, 698036 A url containing the details of these feature requests can be made by --- 160,164 ---- The following feature requests tracked via the Sourceforge system were added: ! 789916, 698036, 796832, 791319 A url containing the details of these feature requests can be made by *************** *** 165,169 **** =================== The following patches tracked via the Sourceforge system were integrated: ! 791254, 790615, 788001, 769981 A url containing the details of these feature requests can be made by --- 170,174 ---- =================== The following patches tracked via the Sourceforge system were integrated: ! 791254, 790615, 788001, 769981, 791393 A url containing the details of these feature requests can be made by From anadelonbrin at users.sourceforge.net Mon Sep 1 03:39:34 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon Sep 1 05:39:39 2003 Subject: [Spambayes-checkins] spambayes/spambayes storage.py,1.25,1.26 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv9393/spambayes Modified Files: storage.py Log Message: Add the ability to get a list of the words (i.e. keys) in a SQL based classifier. Index: storage.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v retrieving revision 1.25 retrieving revision 1.26 diff -C2 -d -r1.25 -r1.26 *** storage.py 1 Sep 2003 09:05:51 -0000 1.25 --- storage.py 1 Sep 2003 09:39:31 -0000 1.26 *************** *** 377,380 **** --- 377,392 ---- self._delete_row(word) + def _wordinfokeys(self): + c = self.cursor() + c.execute("select word from bayes") + rows = self.fetchall(c) + # There is probably some clever way to do this with map or + # something, but I don't know what it is. We want the first + # element from all the items in 'rows' + keys = [] + for r in rows: + keys.append(r[0]) + return keys + class PGClassifier(SQLClassifier): From anadelonbrin at users.sourceforge.net Mon Sep 1 04:05:00 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon Sep 1 06:07:39 2003 Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py,NONE,1.1 Message-ID: Update of /cvsroot/spambayes/spambayes/windows In directory sc8-pr-cvs1:/tmp/cvs-serv13593/windows Added Files: pop3proxy_tray.py Log Message: A basic shell for a pop3proxy tray application, which can open up the configuration page, and start and stop the proxy. The start/stop code needs a lot more work, as part of the pop3proxy binary process. Note that this will cause Python 2.3 to unexpectedly terminate, but will work fine in Python 2.2. See [ 798452 ] win32gui_taskbar.py broken in Python 2.3 --- NEW FILE: pop3proxy_tray.py --- #!/usr/bin/env python """A script to provide an icon in the Windows taskbar tray to control the POP3 proxy. """ # This module is part of the spambayes project, which is Copyright 2002-3 # The Python Software Foundation and is covered by the Python Software # Foundation license. __author__ = "Tony Meyer " __credits__ = "Mark Hammond, all the Spambayes folk." try: True, False except NameError: # Maintain compatibility with Python 2.2 True, False = 1, 0 # Heavily based on the win32gui_taskbar.py demo from Mark Hammond's # win32 extensions. import os import sys import webbrowser import win32con from win32api import * from win32gui import * # Allow for those without SpamBayes on the PYTHONPATH sys.path.insert(-1, os.getcwd()) sys.path.insert(-1, os.path.dirname(os.getcwd())) import pop3proxy from spambayes import Dibbler from spambayes.Options import options class MainWindow(object): def __init__(self): # The ordering here is important - it is the order that they will # appear in the menu. As dicts don't have an order, this means # that the order is controlled by the id. Any items were the # function is None will appear as separators. self.control_functions = {1024 : ("Start SpamBayes", self.StartStop), 1025 : ("-", None), 1026 : ("View information ...", self.OpenInterface), 1027 : ("Configure ...", self.OpenConfig), 1028 : ("-", None), 1029 : ("Exit SpamBayes", self.OnExit), } message_map = { win32con.WM_DESTROY: self.OnDestroy, win32con.WM_COMMAND: self.OnCommand, win32con.WM_USER+20 : self.OnTaskbarNotify, } # Register the Window class. wc = WNDCLASS() hinst = wc.hInstance = GetModuleHandle(None) wc.lpszClassName = "SpambayesTaskbar" wc.style = win32con.CS_VREDRAW | win32con.CS_HREDRAW; wc.hCursor = LoadCursor( 0, win32con.IDC_ARROW ) wc.hbrBackground = win32con.COLOR_WINDOW wc.lpfnWndProc = message_map classAtom = RegisterClass(wc) # Create the Window. style = win32con.WS_OVERLAPPED | win32con.WS_SYSMENU self.hwnd = CreateWindow(classAtom, "SpamBayes", style, 0, 0, win32con.CW_USEDEFAULT, win32con.CW_USEDEFAULT, 0, 0, hinst, None) UpdateWindow(self.hwnd) # Try and find a custom icon # XXX This needs to be done, but first someone needs to make a wee # XXX spambayes icon iconPathName = os.path.abspath(os.path.join( os.path.split(sys.executable)[0], "pyc.ico" )) if not os.path.isfile(iconPathName): # Look in the source tree. iconPathName = os.path.abspath(os.path.join( os.path.split(sys.executable)[0], "..\\PC\\pyc.ico" )) if os.path.isfile(iconPathName): icon_flags = win32con.LR_LOADFROMFILE | win32con.LR_DEFAULTSIZE hicon = LoadImage(hinst, iconPathName, win32con.IMAGE_ICON, 0, 0, icon_flags) else: print "Can't find a spambayes icon file - using default" hicon = LoadIcon(0, win32con.IDI_APPLICATION) flags = NIF_ICON | NIF_MESSAGE | NIF_TIP nid = (self.hwnd, 0, flags, win32con.WM_USER+20, hicon, "SpamBayes") Shell_NotifyIcon(NIM_ADD, nid) # Start up pop3proxy # XXX This needs to be finished off. # XXX This should determine if we are using the service, and if so # XXX start that, and if not kick pop3proxy off in a separate thread. #pop3proxy.prepare(state=pop3proxy.state) #pop3proxy.start(pop3proxy.state) #self.started = True def OnDestroy(self, hwnd, msg, wparam, lparam): nid = (self.hwnd, 0) Shell_NotifyIcon(NIM_DELETE, nid) PostQuitMessage(0) def OnTaskbarNotify(self, hwnd, msg, wparam, lparam): if lparam==win32con.WM_LBUTTONUP: # We ignore left clicks pass elif lparam==win32con.WM_LBUTTONDBLCLK: # Default behaviour is to open up the web interface # XXX This should be set as the default (which then means bold # XXX text) through the win32 calls, but win32all doesn't # XXX include SetDefault(), which it needs to... self.OpenInterface() elif lparam==win32con.WM_RBUTTONUP: menu = CreatePopupMenu() ids = self.control_functions.keys() ids.sort() for id in ids: (wording, function) = self.control_functions[id] if function: AppendMenu( menu, win32con.MF_STRING, id, wording) else: AppendMenu( menu, win32con.MF_SEPARATOR, id, wording) pos = GetCursorPos() SetForegroundWindow(self.hwnd) TrackPopupMenu(menu, win32con.TPM_LEFTALIGN, pos[0], pos[1], 0, self.hwnd, None) PostMessage(self.hwnd, win32con.WM_NULL, 0, 0) return 1 def OnCommand(self, hwnd, msg, wparam, lparam): id = LOWORD(wparam) try: unused, function = self.control_functions[id] except KeyError: print "Unknown command -", id function() def OnExit(self): DestroyWindow(self.hwnd) sys.exit() def StartStop(self): # XXX This needs to be finished off. # XXX This should determine if we are using the service, and if so # XXX start/stop that, and if not kick pop3proxy off in a separate # XXX thread, or stop the thread that was started. if self.started: pop3proxy.stop(pop3proxy.state) self.started = False else: pop3proxy.start(pop3proxy.state) self.started = True def OpenInterface(self): webbrowser.open_new("http://localhost:%d/" % \ (options["html_ui", "port"],)) def OpenConfig(self): webbrowser.open_new("http://localhost:%d/config" % \ (options["html_ui", "port"],)) def main(): w = MainWindow() PumpMessages() if __name__=='__main__': main() From anadelonbrin at users.sourceforge.net Mon Sep 1 04:33:50 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon Sep 1 06:33:53 2003 Subject: [Spambayes-checkins] spambayes pop3proxy.py, 1.96, 1.97 smtpproxy.py, 1.13, 1.14 Message-ID: Update of /cvsroot/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv18001 Modified Files: pop3proxy.py smtpproxy.py Log Message: Fix an (unreported) bug where restoring the defaults would not take effect until restarting the proxy/filter. Fix [ spambayes-Bugs-796996 ] smtp server not started until restart, which should also fix Richie's bug about the smtpproxy not stopping when it was meant to, and I think also Bruce Richardson's too many open connections problem. Index: pop3proxy.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/pop3proxy.py,v retrieving revision 1.96 retrieving revision 1.97 diff -C2 -d -r1.96 -r1.97 *** pop3proxy.py 31 Aug 2003 00:29:07 -0000 1.96 --- pop3proxy.py 1 Sep 2003 10:33:48 -0000 1.97 *************** *** 710,715 **** global state state = State() - state.buildServerStrings() - state.createWorkers() # Close the existing listeners and create new ones. This won't --- 710,713 ---- *************** *** 719,723 **** --- 717,724 ---- proxy.close() del proxyListeners[:] + + prepare(state) _createProxies(state.servers, state.proxyPorts) + return state *************** *** 740,744 **** import smtpproxy servers, proxyPorts = smtpproxy.LoadServerInfo() ! smtpproxy.CreateProxies(servers, proxyPorts, state) # setup info for the web interface --- 741,746 ---- import smtpproxy servers, proxyPorts = smtpproxy.LoadServerInfo() ! proxyListeners.extend(smtpproxy.CreateProxies(servers, proxyPorts, ! state)) # setup info for the web interface Index: smtpproxy.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/smtpproxy.py,v retrieving revision 1.13 retrieving revision 1.14 diff -C2 -d -r1.13 -r1.14 *** smtpproxy.py 1 Sep 2003 05:06:37 -0000 1.13 --- smtpproxy.py 1 Sep 2003 10:33:48 -0000 1.14 *************** *** 144,148 **** from spambayes.Options import options from pop3proxy import _addressPortStr, ServerLineReader ! from pop3proxy import _addressAndPort, proxyListeners class SMTPProxyBase(Dibbler.BrighterAsyncChat): --- 144,148 ---- from spambayes.Options import options from pop3proxy import _addressPortStr, ServerLineReader ! from pop3proxy import _addressAndPort class SMTPProxyBase(Dibbler.BrighterAsyncChat): *************** *** 516,523 **** --- 516,525 ---- if not isinstance(trainer, SMTPTrainer): trainer = SMTPTrainer(trainer.bayes, trainer) + proxyListeners = [] for (server, serverPort), proxyPort in zip(servers, proxyPorts): listener = BayesSMTPProxyListener(server, serverPort, proxyPort, trainer) proxyListeners.append(listener) + return proxyListeners def main(): *************** *** 561,566 **** servers, proxyPorts = LoadServerInfo() trainer = SMTPTrainer(classifier) ! CreateProxies(servers, proxyPorts, trainer) Dibbler.run() if __name__ == '__main__': --- 563,569 ---- servers, proxyPorts = LoadServerInfo() trainer = SMTPTrainer(classifier) ! proxyListeners = CreateProxies(servers, proxyPorts, trainer) Dibbler.run() + if __name__ == '__main__': From anadelonbrin at users.sourceforge.net Mon Sep 1 04:33:51 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon Sep 1 06:33:57 2003 Subject: [Spambayes-checkins] spambayes/spambayes UserInterface.py, 1.22, 1.23 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv18001/spambayes Modified Files: UserInterface.py Log Message: Fix an (unreported) bug where restoring the defaults would not take effect until restarting the proxy/filter. Fix [ spambayes-Bugs-796996 ] smtp server not started until restart, which should also fix Richie's bug about the smtpproxy not stopping when it was meant to, and I think also Bruce Richardson's too many open connections problem. Index: UserInterface.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/UserInterface.py,v retrieving revision 1.22 retrieving revision 1.23 diff -C2 -d -r1.22 -r1.23 *** UserInterface.py 1 Sep 2003 09:05:51 -0000 1.22 --- UserInterface.py 1 Sep 2003 10:33:48 -0000 1.23 *************** *** 713,716 **** --- 713,717 ---- self.writeOKHeaders('text/html') self.write(html) + self.reReadOptions() def verifyInput(self, parms, pmap): From anadelonbrin at users.sourceforge.net Mon Sep 1 04:36:16 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon Sep 1 06:36:19 2003 Subject: [Spambayes-checkins] spambayes CHANGELOG.txt, 1.15, 1.16 WHAT_IS_NEW.txt, 1.12, 1.13 Message-ID: Update of /cvsroot/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv18376 Modified Files: CHANGELOG.txt WHAT_IS_NEW.txt Log Message: Bring up to date, and add a clarifying statement about the dates. Index: CHANGELOG.txt =================================================================== RCS file: /cvsroot/spambayes/spambayes/CHANGELOG.txt,v retrieving revision 1.15 retrieving revision 1.16 diff -C2 -d -r1.15 -r1.16 *** CHANGELOG.txt 1 Sep 2003 09:16:08 -0000 1.15 --- CHANGELOG.txt 1 Sep 2003 10:36:14 -0000 1.16 *************** *** 1,4 **** --- 1,7 ---- + [Note that all dates are in English, not American format - i.e. day/month/year] + Alpha Release 5 =============== + Tony Meyer 01/09/2003 Fix [ spambayes-Bugs-796996 ] smtp server not started until restart. Richie Hindle 01/09/2003 Integrated [ 791393 ] HTTP Authentication, which closes #791319. Tony Meyer 01/09/2003 Added [ 796832 ] Word query should show all words starting with certain text Index: WHAT_IS_NEW.txt =================================================================== RCS file: /cvsroot/spambayes/spambayes/WHAT_IS_NEW.txt,v retrieving revision 1.12 retrieving revision 1.13 diff -C2 -d -r1.12 -r1.13 *** WHAT_IS_NEW.txt 1 Sep 2003 09:16:08 -0000 1.12 --- WHAT_IS_NEW.txt 1 Sep 2003 10:36:14 -0000 1.13 *************** *** 149,153 **** 790406, 788008, 787296, 788002, 780612, 784323, 784296, 780819, 780801, 779049, 765912, 777026, 777165, 693387, 690418, 719586, 769346, 761499, ! 769346, 773452, 765042, 760062, 768162, 768221, 797776, 797316 A url containing the details of these bugs can be made by appending the --- 149,153 ---- 790406, 788008, 787296, 788002, 780612, 784323, 784296, 780819, 780801, 779049, 765912, 777026, 777165, 693387, 690418, 719586, 769346, 761499, ! 769346, 773452, 765042, 760062, 768162, 768221, 797776, 797316, 796996 A url containing the details of these bugs can be made by appending the From richiehindle at users.sourceforge.net Mon Sep 1 15:07:34 2003 From: richiehindle at users.sourceforge.net (Richie Hindle) Date: Mon Sep 1 17:07:38 2003 Subject: [Spambayes-checkins] spambayes/spambayes storage.py, 1.26, 1.27 classifier.py, 1.7, 1.8 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv1240 Modified Files: storage.py classifier.py Log Message: Bug #797890: After training on a message, ensure that the classifier's state (nspam, nham) is written to the database. Otherwise, if training goes wrong, you can get a wordinfo whose count is greater than nspam/nham - for instance, when training on a 100-message mailbox file, if the 99th message caused an exception you could get a clue with a ham count of 99 but an nham of 0 (or whatever it was when you started training). Index: storage.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v retrieving revision 1.26 retrieving revision 1.27 diff -C2 -d -r1.26 -r1.27 *** storage.py 1 Sep 2003 09:39:31 -0000 1.26 --- storage.py 1 Sep 2003 21:07:32 -0000 1.27 *************** *** 214,221 **** self.changed_words = {} # Update the global state, then do the actual save. self.db[self.statekey] = (classifier.PICKLE_VERSION, self.nspam, self.nham) - self.db.sync() def _wordinfoget(self, word): if isinstance(word, unicode): --- 214,230 ---- self.changed_words = {} # Update the global state, then do the actual save. + self._write_state_key() + self.db.sync() + + def _write_state_key(self): self.db[self.statekey] = (classifier.PICKLE_VERSION, self.nspam, self.nham) + def _post_training(self): + """This is called after training on a wordstream. We ensure that the + database is in a consistent state at this point by writing the state + key.""" + self._write_state_key() + def _wordinfoget(self, word): if isinstance(word, unicode): Index: classifier.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/classifier.py,v retrieving revision 1.7 retrieving revision 1.8 diff -C2 -d -r1.7 -r1.8 *** classifier.py 1 Sep 2003 09:05:51 -0000 1.7 --- classifier.py 1 Sep 2003 21:07:32 -0000 1.8 *************** *** 411,414 **** --- 411,415 ---- self._wordinfoset(word, record) + self._post_training() def _remove_msg(self, wordstream, is_spam): *************** *** 437,440 **** --- 438,449 ---- self._wordinfoset(word, record) + self._post_training() + + def _post_training(self): + """This is called after training on a wordstream. Subclasses might + want to ensure that their databases are in a consistent state at + this point. Introduced to fix bug #797890.""" + pass + def _getclues(self, wordstream): mindist = options.minimum_prob_strength From richiehindle at users.sourceforge.net Mon Sep 1 15:31:46 2003 From: richiehindle at users.sourceforge.net (Richie Hindle) Date: Mon Sep 1 17:31:53 2003 Subject: [Spambayes-checkins] spambayes/spambayes Version.py,1.14,1.15 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv6441 Modified Files: Version.py Log Message: Lots of fixes and improvements to the POP3 proxy and the web interface since May - got to be worth an uprev. Index: Version.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/Version.py,v retrieving revision 1.14 retrieving revision 1.15 diff -C2 -d -r1.14 -r1.15 *** Version.py 9 Aug 2003 03:26:54 -0000 1.14 --- Version.py 1 Sep 2003 21:31:44 -0000 1.15 *************** *** 44,52 **** }, "POP3 Proxy" : { ! "Version": 0.1, ! "Description": "SpamBayes POP3 Proxy Beta1", ! "Date": "May 2003", ! "InterfaceVersion": 0.02, ! "InterfaceDescription": "SpamBayes POP3 Proxy Web Interface Alpha2", "Full Description": """%(Description)s, version %(Version)s (%(Date)s), using %(InterfaceDescription)s, version %(InterfaceVersion)s""", --- 44,52 ---- }, "POP3 Proxy" : { ! "Version": 0.2, ! "Description": "SpamBayes POP3 Proxy Beta2", ! "Date": "September 2003", ! "InterfaceVersion": 0.03, ! "InterfaceDescription": "SpamBayes POP3 Proxy Web Interface Alpha3", "Full Description": """%(Description)s, version %(Version)s (%(Date)s), using %(InterfaceDescription)s, version %(InterfaceVersion)s""", From richiehindle at users.sourceforge.net Mon Sep 1 15:36:45 2003 From: richiehindle at users.sourceforge.net (Richie Hindle) Date: Mon Sep 1 17:36:48 2003 Subject: [Spambayes-checkins] spambayes CHANGELOG.txt, 1.16, 1.17 WHAT_IS_NEW.txt, 1.13, 1.14 Message-ID: Update of /cvsroot/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv7277 Modified Files: CHANGELOG.txt WHAT_IS_NEW.txt Log Message: Update for 797890 fix. Index: CHANGELOG.txt =================================================================== RCS file: /cvsroot/spambayes/spambayes/CHANGELOG.txt,v retrieving revision 1.16 retrieving revision 1.17 diff -C2 -d -r1.16 -r1.17 *** CHANGELOG.txt 1 Sep 2003 10:36:14 -0000 1.16 --- CHANGELOG.txt 1 Sep 2003 21:36:42 -0000 1.17 *************** *** 3,6 **** --- 3,7 ---- Alpha Release 5 =============== + Richie Hindle 01/09/2003 Fix [ 797890 ] "assert hamcount <= nham" problem. Tony Meyer 01/09/2003 Fix [ spambayes-Bugs-796996 ] smtp server not started until restart. Richie Hindle 01/09/2003 Integrated [ 791393 ] HTTP Authentication, which closes #791319. Index: WHAT_IS_NEW.txt =================================================================== RCS file: /cvsroot/spambayes/spambayes/WHAT_IS_NEW.txt,v retrieving revision 1.13 retrieving revision 1.14 diff -C2 -d -r1.13 -r1.14 *** WHAT_IS_NEW.txt 1 Sep 2003 10:36:14 -0000 1.13 --- WHAT_IS_NEW.txt 1 Sep 2003 21:36:43 -0000 1.14 *************** *** 41,44 **** --- 41,45 ---- POP3 Proxy / SMTP Proxy / POP3 Proxy Service -------------------------------------------- + o Fixed "assert hamcount <= nham" problem. o Starting and stopping the POP3 Proxy service (for Windows NT, Windows 2000 and Windows XP users) has been improved. Most noticeably, this From xenogeist at users.sourceforge.net Mon Sep 1 15:43:25 2003 From: xenogeist at users.sourceforge.net (Adam Walker) Date: Mon Sep 1 17:43:39 2003 Subject: [Spambayes-checkins] spambayes/windows/resources - New directory Message-ID: Update of /cvsroot/spambayes/spambayes/windows/resources In directory sc8-pr-cvs1:/tmp/cvs-serv8327/resources Log Message: Directory /cvsroot/spambayes/spambayes/windows/resources added to the repository From xenogeist at users.sourceforge.net Mon Sep 1 15:46:28 2003 From: xenogeist at users.sourceforge.net (Adam Walker) Date: Mon Sep 1 17:46:32 2003 Subject: [Spambayes-checkins] spambayes/windows/resources dialogs.h, NONE, 1.1 dialogs.rc, NONE, 1.1 sbicon.ico, NONE, 1.1 Message-ID: Update of /cvsroot/spambayes/spambayes/windows/resources In directory sc8-pr-cvs1:/tmp/cvs-serv8836/windows/resources Added Files: dialogs.h dialogs.rc sbicon.ico Log Message: Fixed Tony's tray program to work around Python 2.3. Added code to run the proxy in a seperate thread (I'll leave interacting with a service for another day). Includes the icon voted most likely change. --- NEW FILE: dialogs.h --- //{{NO_DEPENDENCIES}} // Microsoft Visual C++ generated include file. // Used by dialogs.rc // #define IDD_MANAGER 101 #define IDI_SBICON 102 #define IDC_PROGRESS 1000 #define IDC_PROGRESS_TEXT 1001 #define IDC_STATIC_HAM 1002 #define IDC_STATIC_SPAM 1003 #define IDC_BROWSE_HAM 1004 #define IDC_BROWSE_SPAM 1005 #define IDC_START 1006 #define IDC_BUT_REBUILD 1007 #define IDC_BUT_RESCORE 1008 #define IDC_VERSION 1009 #define IDC_BUT_TRAIN_FROM_SPAM_FOLDER 1010 #define IDC_BUT_TRAIN_TO_SPAM_FOLDER 1011 #define IDC_BUT_TRAIN_NOW 1012 #define IDC_BUT_FILTER_ENABLE 1013 #define IDC_FILTER_STATUS 1014 #define IDC_BUT_FILTER_NOW 1015 #define IDC_BUT_FILTER_DEFINE 1016 #define IDC_BUT_ABOUT 1017 #define IDC_BUT_ACT_SCORE 1018 #define IDC_BUT_ACT_ALL 1019 #define IDC_BUT_UNREAD 1020 #define IDC_BUT_UNSEEN 1021 #define IDC_BUT_FILTERNOW 1022 #define IDC_SLIDER_CERTAIN 1023 #define IDC_EDIT_CERTAIN 1024 #define IDC_ACTION_CERTAIN 1025 #define IDC_TOFOLDER_CERTAIN 1026 #define IDC_FOLDER_CERTAIN 1027 #define IDC_BROWSE_CERTAIN 1028 #define IDC_SLIDER_UNSURE 1029 #define IDC_EDIT_UNSURE 1030 #define IDC_ACTION_UNSURE 1031 #define IDC_TOFOLDER_UNSURE 1032 #define IDC_FOLDER_UNSURE 1033 #define IDC_BROWSE_UNSURE 1034 #define IDC_TRAINING_STATUS 1035 #define IDC_FOLDER_NAMES 1036 #define IDC_BROWSE 1037 #define IDC_FOLDER_WATCH 1038 #define IDC_BROWSE_WATCH 1039 #define IDC_LIST_FOLDERS 1040 #define IDC_BUT_SEARCHSUB 1041 #define IDC_BUT_CLEARALL 1042 #define IDC_STATUS1 1043 #define IDC_STATUS2 1044 #define IDC_BUT_CLEARALL2 1045 #define IDC_BUT_NEW 1046 #define IDC_MARK_SPAM_AS_READ 1047 #define IDC_SAVE_SPAM_SCORE 1048 #define IDC_MARK_UNSURE_AS_READ 1051 #define IDC_ADVANCED_BTN 1055 #define IDC_DELAY1_SLIDER 1056 #define IDC_DELAY1_TEXT 1057 #define IDC_DELAY2_SLIDER 1058 #define IDC_DELAY2_TEXT 1059 #define IDC_INBOX_TIMER_ONLY 1060 #define IDC_VERBOSE_LOG 1061 #define IDB_SBLOGO 1062 #define IDC_LOGO_GRAPHIC 1063 #define IDC_USE_DELAY1 1064 #define IDC_USE_DELAY2 1065 #define IDC_TAB 1068 #define IDC_BACK_BTN 1069 #define IDC_BUT_WIZARD 1070 #define IDC_SHOW_DATA_FOLDER 1071 #define IDC_ABOUT_BTN 1072 #define IDC_BUT_RESET 1073 #define IDC_DEL_SPAM_RS 1074 #define IDC_RECOVER_RS 1075 #define IDC_HIDDEN 1076 #define IDC_FORWARD_BTN 1077 #define IDC_PAGE_PLACEHOLDER 1078 #define IDC_SHOW_DATA_FOLDER2 1079 #define IDC_BUT_SHOW_DIAGNOSTICS 1080 #define IDC_BUT_PREPARATION 1081 #define IDC_FOLDER_HAM 1083 #define IDC_BUT_UNTRAINED 1088 #define IDC_BUT_TRAIN 1089 #define IDC_BUT_TIMER_ENABLED 1091 #define IDC_WIZ_GRAPHIC 1092 #define IDC_BUT_VIEW_LOG 1093 // Next default values for new objects // #ifdef APSTUDIO_INVOKED #ifndef APSTUDIO_READONLY_SYMBOLS #define _APS_NEXT_RESOURCE_VALUE 103 #define _APS_NEXT_COMMAND_VALUE 40001 #define _APS_NEXT_CONTROL_VALUE 1094 #define _APS_NEXT_SYMED_VALUE 101 #endif #endif --- NEW FILE: dialogs.rc --- // Microsoft Visual C++ generated resource script. // #include "dialogs.h" #define APSTUDIO_READONLY_SYMBOLS ///////////////////////////////////////////////////////////////////////////// // // Generated from the TEXTINCLUDE 2 resource. // #include "winres.h" // spambayes dialog definitions ///////////////////////////////////////////////////////////////////////////// #undef APSTUDIO_READONLY_SYMBOLS ///////////////////////////////////////////////////////////////////////////// // English (U.S.) resources #if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) #ifdef _WIN32 LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US #pragma code_page(1252) #endif //_WIN32 ///////////////////////////////////////////////////////////////////////////// // // Dialog // IDD_MANAGER DIALOGEX 0, 0, 275, 260 STYLE DS_SETFONT | DS_MODALFRAME | WS_POPUP | WS_CAPTION | WS_SYSMENU EXSTYLE WS_EX_CONTEXTHELP CAPTION "SpamBayes" FONT 8, "Tahoma", 400, 0, 0x0 BEGIN DEFPUSHBUTTON "Close",IDOK,216,239,50,14 PUSHBUTTON "Cancel",IDCANCEL,155,239,50,14,NOT WS_VISIBLE CONTROL "",IDC_TAB,"SysTabControl32",0x0,8,7,258,228 PUSHBUTTON "About",IDC_ABOUT_BTN,9,239,50,14 END #ifdef APSTUDIO_INVOKED ///////////////////////////////////////////////////////////////////////////// // // TEXTINCLUDE // 1 TEXTINCLUDE BEGIN "dialogs.h\0" END 2 TEXTINCLUDE BEGIN "#include ""winres.h""\r\n" "// spambayes dialog definitions\r\n" "\0" END 3 TEXTINCLUDE BEGIN "\r\n" "\0" END #endif // APSTUDIO_INVOKED ///////////////////////////////////////////////////////////////////////////// // // Icon // // Icon with lowest ID value placed first to ensure application icon // remains consistent on all systems. IDI_SBICON ICON "sbicon.ico" #endif // English (U.S.) resources ///////////////////////////////////////////////////////////////////////////// #ifndef APSTUDIO_INVOKED ///////////////////////////////////////////////////////////////////////////// // // Generated from the TEXTINCLUDE 3 resource. // ///////////////////////////////////////////////////////////////////////////// #endif // not APSTUDIO_INVOKED --- NEW FILE: sbicon.ico --- (This appears to be a binary file; contents omitted.) From xenogeist at users.sourceforge.net Mon Sep 1 15:46:28 2003 From: xenogeist at users.sourceforge.net (Adam Walker) Date: Mon Sep 1 17:46:33 2003 Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py,1.1,1.2 Message-ID: Update of /cvsroot/spambayes/spambayes/windows In directory sc8-pr-cvs1:/tmp/cvs-serv8836/windows Modified Files: pop3proxy_tray.py Log Message: Fixed Tony's tray program to work around Python 2.3. Added code to run the proxy in a seperate thread (I'll leave interacting with a service for another day). Includes the icon voted most likely change. Index: pop3proxy_tray.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_tray.py,v retrieving revision 1.1 retrieving revision 1.2 diff -C2 -d -r1.1 -r1.2 *** pop3proxy_tray.py 1 Sep 2003 10:04:58 -0000 1.1 --- pop3proxy_tray.py 1 Sep 2003 21:46:26 -0000 1.2 *************** *** 9,13 **** # Foundation license. ! __author__ = "Tony Meyer " __credits__ = "Mark Hammond, all the Spambayes folk." --- 9,13 ---- # Foundation license. ! __author__ = "Tony Meyer , Adam Walker" __credits__ = "Mark Hammond, all the Spambayes folk." *************** *** 24,27 **** --- 24,28 ---- import sys import webbrowser + import thread import win32con *************** *** 37,40 **** --- 38,43 ---- from spambayes.Options import options + WM_TASKBAR_NOTIFY = win32con.WM_USER + 20 + class MainWindow(object): def __init__(self): *************** *** 53,80 **** win32con.WM_DESTROY: self.OnDestroy, win32con.WM_COMMAND: self.OnCommand, ! win32con.WM_USER+20 : self.OnTaskbarNotify, } - # Register the Window class. - wc = WNDCLASS() - hinst = wc.hInstance = GetModuleHandle(None) - wc.lpszClassName = "SpambayesTaskbar" - wc.style = win32con.CS_VREDRAW | win32con.CS_HREDRAW; - wc.hCursor = LoadCursor( 0, win32con.IDC_ARROW ) - wc.hbrBackground = win32con.COLOR_WINDOW - wc.lpfnWndProc = message_map - classAtom = RegisterClass(wc) - # Create the Window. ! style = win32con.WS_OVERLAPPED | win32con.WS_SYSMENU ! self.hwnd = CreateWindow(classAtom, "SpamBayes", style, ! 0, 0, win32con.CW_USEDEFAULT, ! win32con.CW_USEDEFAULT, 0, 0, hinst, None) ! UpdateWindow(self.hwnd) # Try and find a custom icon # XXX This needs to be done, but first someone needs to make a wee # XXX spambayes icon ! iconPathName = os.path.abspath(os.path.join( os.path.split(sys.executable)[0], "pyc.ico" )) if not os.path.isfile(iconPathName): # Look in the source tree. --- 56,74 ---- win32con.WM_DESTROY: self.OnDestroy, win32con.WM_COMMAND: self.OnCommand, ! WM_TASKBAR_NOTIFY : self.OnTaskbarNotify, } # Create the Window. ! hinst = GetModuleHandle(None) ! # this will replaced with a real configure dialog later ! # this is mainly to work around not being able to register a window class ! # with python 2.3 ! dialogTemplate = [['SpamBayes', (14, 10, 246, 187), -1865809852 & ~win32con.WS_VISIBLE, None, (8, 'Tahoma')],] ! self.hwnd = CreateDialogIndirect(hinst, dialogTemplate, 0, message_map) # Try and find a custom icon # XXX This needs to be done, but first someone needs to make a wee # XXX spambayes icon ! iconPathName = os.path.abspath( "resources\\sbicon.ico" ) if not os.path.isfile(iconPathName): # Look in the source tree. *************** *** 88,93 **** flags = NIF_ICON | NIF_MESSAGE | NIF_TIP ! nid = (self.hwnd, 0, flags, win32con.WM_USER+20, hicon, "SpamBayes") Shell_NotifyIcon(NIM_ADD, nid) # Start up pop3proxy --- 82,88 ---- flags = NIF_ICON | NIF_MESSAGE | NIF_TIP ! nid = (self.hwnd, 0, flags, WM_TASKBAR_NOTIFY, hicon, "SpamBayes") Shell_NotifyIcon(NIM_ADD, nid) + self.started = False # Start up pop3proxy *************** *** 95,106 **** # XXX This should determine if we are using the service, and if so # XXX start that, and if not kick pop3proxy off in a separate thread. ! #pop3proxy.prepare(state=pop3proxy.state) ! #pop3proxy.start(pop3proxy.state) ! #self.started = True def OnDestroy(self, hwnd, msg, wparam, lparam): ! nid = (self.hwnd, 0) ! Shell_NotifyIcon(NIM_DELETE, nid) ! PostQuitMessage(0) def OnTaskbarNotify(self, hwnd, msg, wparam, lparam): --- 90,100 ---- # XXX This should determine if we are using the service, and if so # XXX start that, and if not kick pop3proxy off in a separate thread. ! pop3proxy.prepare(state=pop3proxy.state) ! self.StartProxyThread() def OnDestroy(self, hwnd, msg, wparam, lparam): ! nid = (self.hwnd, 0) ! Shell_NotifyIcon(NIM_DELETE, nid) ! PostQuitMessage(0) def OnTaskbarNotify(self, hwnd, msg, wparam, lparam): *************** *** 142,145 **** --- 136,144 ---- DestroyWindow(self.hwnd) sys.exit() + + def StartProxyThread(self): + args = (pop3proxy.state,) + thread.start_new_thread(pop3proxy.start, args) + self.started = True def StartStop(self): *************** *** 152,157 **** self.started = False else: ! pop3proxy.start(pop3proxy.state) ! self.started = True def OpenInterface(self): --- 151,155 ---- self.started = False else: ! self.StartProxyThread() def OpenInterface(self): From xenogeist at users.sourceforge.net Mon Sep 1 15:56:22 2003 From: xenogeist at users.sourceforge.net (Adam Walker) Date: Mon Sep 1 17:56:33 2003 Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py,1.2,1.3 Message-ID: Update of /cvsroot/spambayes/spambayes/windows In directory sc8-pr-cvs1:/tmp/cvs-serv10626/windows Modified Files: pop3proxy_tray.py Log Message: Be nice and stop the proxy before exiting the app. Index: pop3proxy_tray.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_tray.py,v retrieving revision 1.2 retrieving revision 1.3 diff -C2 -d -r1.2 -r1.3 *** pop3proxy_tray.py 1 Sep 2003 21:46:26 -0000 1.2 --- pop3proxy_tray.py 1 Sep 2003 21:56:20 -0000 1.3 *************** *** 134,137 **** --- 134,140 ---- def OnExit(self): + if self.started: + pop3proxy.stop(pop3proxy.state) + self.started = False DestroyWindow(self.hwnd) sys.exit() From anadelonbrin at users.sourceforge.net Mon Sep 1 17:58:24 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon Sep 1 19:58:28 2003 Subject: [Spambayes-checkins] spambayes WHAT_IS_NEW.txt,1.14,1.15 Message-ID: Update of /cvsroot/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv30418 Modified Files: WHAT_IS_NEW.txt Log Message: I'd forgotten about the version info :) smtpproxy has definately gone up a version, and the same for the imapfilter interface, although not the filter itself. Index: WHAT_IS_NEW.txt =================================================================== RCS file: /cvsroot/spambayes/spambayes/WHAT_IS_NEW.txt,v retrieving revision 1.14 retrieving revision 1.15 diff -C2 -d -r1.14 -r1.15 *** WHAT_IS_NEW.txt 1 Sep 2003 21:36:43 -0000 1.14 --- WHAT_IS_NEW.txt 1 Sep 2003 23:58:22 -0000 1.15 *************** *** 150,154 **** 790406, 788008, 787296, 788002, 780612, 784323, 784296, 780819, 780801, 779049, 765912, 777026, 777165, 693387, 690418, 719586, 769346, 761499, ! 769346, 773452, 765042, 760062, 768162, 768221, 797776, 797316, 796996 A url containing the details of these bugs can be made by appending the --- 150,155 ---- 790406, 788008, 787296, 788002, 780612, 784323, 784296, 780819, 780801, 779049, 765912, 777026, 777165, 693387, 690418, 719586, 769346, 761499, ! 769346, 773452, 765042, 760062, 768162, 768221, 797776, 797316, 796996, ! 797890 A url containing the details of these bugs can be made by appending the From anadelonbrin at users.sourceforge.net Mon Sep 1 17:58:25 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon Sep 1 19:58:30 2003 Subject: [Spambayes-checkins] spambayes/spambayes Version.py,1.15,1.16 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv30418/spambayes Modified Files: Version.py Log Message: I'd forgotten about the version info :) smtpproxy has definately gone up a version, and the same for the imapfilter interface, although not the filter itself. Index: Version.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/Version.py,v retrieving revision 1.15 retrieving revision 1.16 diff -C2 -d -r1.15 -r1.16 *** Version.py 1 Sep 2003 21:31:44 -0000 1.15 --- Version.py 1 Sep 2003 23:58:22 -0000 1.16 *************** *** 59,65 **** }, "SMTP Proxy" : { ! "Version": 0.01, ! "Description": "SpamBayes SMTP Proxy Alpha1", ! "Date": "May 2003", "Full Description": "%(Description)s, version %(Version)s (%(Date)s)", }, --- 59,65 ---- }, "SMTP Proxy" : { ! "Version": 0.02, ! "Description": "SpamBayes SMTP Proxy Alpha2", ! "Date": "September 2003", "Full Description": "%(Description)s, version %(Version)s (%(Date)s)", }, *************** *** 68,73 **** "Description": "SpamBayes IMAP Filter Alpha1", "Date": "May 2003", ! "InterfaceVersion": 0.01, ! "InterfaceDescription": "SpamBayes IMAP Filter Web Interface Alpha1", "Full Description": """%(Description)s, version %(Version)s (%(Date)s), using %(InterfaceDescription)s, version %(InterfaceVersion)s""", --- 68,73 ---- "Description": "SpamBayes IMAP Filter Alpha1", "Date": "May 2003", ! "InterfaceVersion": 0.02, ! "InterfaceDescription": "SpamBayes IMAP Filter Web Interface Alpha2", "Full Description": """%(Description)s, version %(Version)s (%(Date)s), using %(InterfaceDescription)s, version %(InterfaceVersion)s""", From anadelonbrin at users.sourceforge.net Mon Sep 1 18:15:14 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon Sep 1 20:15:18 2003 Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py,1.3,1.4 Message-ID: Update of /cvsroot/spambayes/spambayes/windows In directory sc8-pr-cvs1:/tmp/cvs-serv383/windows Modified Files: pop3proxy_tray.py Log Message: Remove assumptions about which directory we are running in when finding the icon, and get rid of the code to deal with not finding it. Update the menu to reflect the current start/stop status. Index: pop3proxy_tray.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_tray.py,v retrieving revision 1.3 retrieving revision 1.4 diff -C2 -d -r1.3 -r1.4 *** pop3proxy_tray.py 1 Sep 2003 21:56:20 -0000 1.3 --- pop3proxy_tray.py 2 Sep 2003 00:15:12 -0000 1.4 *************** *** 40,43 **** --- 40,45 ---- WM_TASKBAR_NOTIFY = win32con.WM_USER + 20 + START_STOP_ID = 1024 + class MainWindow(object): def __init__(self): *************** *** 46,50 **** # that the order is controlled by the id. Any items were the # function is None will appear as separators. ! self.control_functions = {1024 : ("Start SpamBayes", self.StartStop), 1025 : ("-", None), 1026 : ("View information ...", self.OpenInterface), --- 48,52 ---- # that the order is controlled by the id. Any items were the # function is None will appear as separators. ! self.control_functions = {START_STOP_ID : ("Stop SpamBayes", self.StartStop), 1025 : ("-", None), 1026 : ("View information ...", self.OpenInterface), *************** *** 61,83 **** # Create the Window. hinst = GetModuleHandle(None) ! # this will replaced with a real configure dialog later ! # this is mainly to work around not being able to register a window class ! # with python 2.3 ! dialogTemplate = [['SpamBayes', (14, 10, 246, 187), -1865809852 & ~win32con.WS_VISIBLE, None, (8, 'Tahoma')],] ! self.hwnd = CreateDialogIndirect(hinst, dialogTemplate, 0, message_map) ! # Try and find a custom icon ! # XXX This needs to be done, but first someone needs to make a wee ! # XXX spambayes icon ! iconPathName = os.path.abspath( "resources\\sbicon.ico" ) ! if not os.path.isfile(iconPathName): ! # Look in the source tree. ! iconPathName = os.path.abspath(os.path.join( os.path.split(sys.executable)[0], "..\\PC\\pyc.ico" )) if os.path.isfile(iconPathName): icon_flags = win32con.LR_LOADFROMFILE | win32con.LR_DEFAULTSIZE ! hicon = LoadImage(hinst, iconPathName, win32con.IMAGE_ICON, 0, 0, icon_flags) ! else: ! print "Can't find a spambayes icon file - using default" ! hicon = LoadIcon(0, win32con.IDI_APPLICATION) flags = NIF_ICON | NIF_MESSAGE | NIF_TIP --- 63,85 ---- # Create the Window. hinst = GetModuleHandle(None) ! # This will replaced with a real configure dialog later ! # This is mainly to work around not being able to register a window ! # class with Python 2.3 ! dialogTemplate = [['SpamBayes', (14, 10, 246, 187), ! -1865809852 & ~win32con.WS_VISIBLE, None, ! (8, 'Tahoma')],] ! self.hwnd = CreateDialogIndirect(hinst, dialogTemplate, 0, ! message_map) ! # Get the custom icon ! iconPathName = "%s\\windows\\resources\\sbicon.ico" % \ ! (os.path.dirname(pop3proxy.__file__),) ! # When 1.0a6 is released, the above line will need to change to: ! ## iconPathName = "%s\\..\\windows\\resources\\sbicon.ico" % \ ! ## (os.path.dirname(pop3proxy.__file__),) if os.path.isfile(iconPathName): icon_flags = win32con.LR_LOADFROMFILE | win32con.LR_DEFAULTSIZE ! hicon = LoadImage(hinst, iconPathName, win32con.IMAGE_ICON, 0, ! 0, icon_flags) flags = NIF_ICON | NIF_MESSAGE | NIF_TIP *************** *** 153,158 **** --- 155,164 ---- pop3proxy.stop(pop3proxy.state) self.started = False + self.control_functions[START_STOP_ID] = ("Start SpamBayes", + self.StartStop) else: self.StartProxyThread() + self.control_functions[START_STOP_ID] = ("Stop SpamBayes", + self.StartStop) def OpenInterface(self): From anadelonbrin at users.sourceforge.net Mon Sep 1 19:03:19 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon Sep 1 21:03:22 2003 Subject: [Spambayes-checkins] spambayes imapfilter.py, 1.51, 1.52 smtpproxy.py, 1.14, 1.15 Message-ID: Update of /cvsroot/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv7577 Modified Files: imapfilter.py smtpproxy.py Log Message: Clarify a print statement in smtpproxy. imapfilter: correct options names. correct some comments. change some error checking to be more certain that we will raise the error and not imaplib if something goes wrong. address [ 788845 ] imapfilter training fails, hopefully fixing it. Index: imapfilter.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/imapfilter.py,v retrieving revision 1.51 retrieving revision 1.52 diff -C2 -d -r1.51 -r1.52 *** imapfilter.py 13 Aug 2003 06:36:46 -0000 1.51 --- imapfilter.py 2 Sep 2003 01:03:16 -0000 1.52 *************** *** 60,66 **** much nicer. o IMAP over SSL is untested. ! o Develop a test script, like testtools/pop3proxytest.py that runs ! through some tests (perhaps with a *real* imap server, rather than ! a dummy one). This would make it easier to carry out the tests against each server whenever a change is made. o IMAP supports authentication via other methods than the plain-text --- 60,66 ---- much nicer. o IMAP over SSL is untested. ! o Develop a test script, like spambayes/test/test_pop3proxy.py that ! runs through some tests (perhaps with a *real* imap server, rather ! than a dummy one). This would make it easier to carry out the tests against each server whenever a change is made. o IMAP supports authentication via other methods than the plain-text *************** *** 311,315 **** if self.got_substance: return ! if self.uid is None or self.id is None: print "Cannot get substance of message without an id and an UID" return --- 311,315 ---- if self.got_substance: return ! if not self.uid or not self.id: print "Cannot get substance of message without an id and an UID" return *************** *** 318,321 **** --- 318,322 ---- # the status of the message. Unfortunately, it appears that not # all IMAP servers support this, even though it is in RFC1730 + # Actually, it's not: we should be using BODY.PEEK try: response = imap.uid("FETCH", self.uid, self.rfc822_command) *************** *** 340,345 **** self.epilogue = new_msg.epilogue self._default_type = new_msg._default_type ! if not self.has_key(options["pop3proxy", "mailid_header_name"]): ! self[options["pop3proxy", "mailid_header_name"]] = self.id self.got_substance = True if options["globals", "verbose"]: --- 341,346 ---- self.epilogue = new_msg.epilogue self._default_type = new_msg._default_type ! if not self.has_key(options["Headers", "mailid_header_name"]): ! self[options["Headers", "mailid_header_name"]] = self.id self.got_substance = True if options["globals", "verbose"]: *************** *** 360,364 **** raise RuntimeError, """Can't save a message that doesn't have a folder.""" ! if self.id is None: raise RuntimeError, """Can't save a message that doesn't have an id.""" --- 361,365 ---- raise RuntimeError, """Can't save a message that doesn't have a folder.""" ! if not self.id: raise RuntimeError, """Can't save a message that doesn't have an id.""" *************** *** 403,407 **** imap.SelectFolder(self.folder.name) response = imap.uid("SEARCH", "(UNDELETED HEADER " + \ ! options["pop3proxy", "mailid_header_name"] + \ " " + self.id + ")") self._check(response, 'search') --- 404,408 ---- imap.SelectFolder(self.folder.name) response = imap.uid("SEARCH", "(UNDELETED HEADER " + \ ! options["Headers", "mailid_header_name"] + \ " " + self.id + ")") self._check(response, 'search') *************** *** 416,419 **** --- 417,430 ---- ids = new_id.split(' ') new_id = ids[-1] + # Ok, now we're in trouble if we still haven't found it. + # We make a huge assumption that the new message is the one + # with the highest UID (they are sequential, so this will be + # ok as long as another message hasn't also arrived) + if new_id == "": + response = imap.uid("SEARCH", "ALL") + new_id = response[1][0] + if new_id.find(' ') > -1: + ids = new_id.split(' ') + new_id = ids[-1] self.uid = new_id *************** *** 481,485 **** msg.setFolder(self) msg.uid = key ! r = re.compile(re.escape(options["pop3proxy", "mailid_header_name"]) + \ "\:\s*(\d+(\-\d)?)") --- 492,496 ---- msg.setFolder(self) msg.uid = key ! r = re.compile(re.escape(options["Headers", "mailid_header_name"]) + \ "\:\s*(\d+(\-\d)?)") *************** *** 648,653 **** sys.exit() ! bdbname = options["pop3proxy", "persistent_storage_file"] ! useDBM = options["pop3proxy", "persistent_use_database"] doTrain = False doClassify = False --- 659,664 ---- sys.exit() ! bdbname = options["Storage", "persistent_storage_file"] ! useDBM = options["Storage", "persistent_use_database"] doTrain = False doClassify = False Index: smtpproxy.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/smtpproxy.py,v retrieving revision 1.14 retrieving revision 1.15 diff -C2 -d -r1.14 -r1.15 *** smtpproxy.py 1 Sep 2003 10:33:48 -0000 1.14 --- smtpproxy.py 2 Sep 2003 01:03:17 -0000 1.15 *************** *** 453,457 **** print "Could not find message (%s); perhaps it was " + \ "deleted from the POP3Proxy cache or the IMAP " + \ ! "server." % (id, ) def train_message_in_pop3proxy_cache(self, id, isSpam): --- 453,457 ---- print "Could not find message (%s); perhaps it was " + \ "deleted from the POP3Proxy cache or the IMAP " + \ ! "server. This means that no training was done." % (id, ) def train_message_in_pop3proxy_cache(self, id, isSpam): From anadelonbrin at users.sourceforge.net Mon Sep 1 19:31:06 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon Sep 1 21:31:15 2003 Subject: [Spambayes-checkins] spambayes/spambayes Version.py,1.16,1.17 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv12061/spambayes Modified Files: Version.py Log Message: Use the storage.open_storage function in imapfilter, which means it joins the ranks of giving a nice error when someone tries to use dumbdbm, and users can use the new SQL classifiers. Fix Adam Many's problem with listing folder names (the server was replying with a literal, which we didn't expect). Stop being stingy and give imapfilter a revup, otherwise people will think it hasn't changed at all, when there have been very minor changes. Index: Version.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/Version.py,v retrieving revision 1.16 retrieving revision 1.17 diff -C2 -d -r1.16 -r1.17 *** Version.py 1 Sep 2003 23:58:22 -0000 1.16 --- Version.py 2 Sep 2003 01:31:04 -0000 1.17 *************** *** 65,71 **** }, "IMAP Filter" : { ! "Version": 0.01, ! "Description": "SpamBayes IMAP Filter Alpha1", ! "Date": "May 2003", "InterfaceVersion": 0.02, "InterfaceDescription": "SpamBayes IMAP Filter Web Interface Alpha2", --- 65,71 ---- }, "IMAP Filter" : { ! "Version": 0.02, ! "Description": "SpamBayes IMAP Filter Alpha2", ! "Date": "September 2003", "InterfaceVersion": 0.02, "InterfaceDescription": "SpamBayes IMAP Filter Web Interface Alpha2", From anadelonbrin at users.sourceforge.net Mon Sep 1 19:31:06 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon Sep 1 21:31:17 2003 Subject: [Spambayes-checkins] spambayes WHAT_IS_NEW.txt, 1.15, 1.16 imapfilter.py, 1.52, 1.53 Message-ID: Update of /cvsroot/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv12061 Modified Files: WHAT_IS_NEW.txt imapfilter.py Log Message: Use the storage.open_storage function in imapfilter, which means it joins the ranks of giving a nice error when someone tries to use dumbdbm, and users can use the new SQL classifiers. Fix Adam Many's problem with listing folder names (the server was replying with a literal, which we didn't expect). Stop being stingy and give imapfilter a revup, otherwise people will think it hasn't changed at all, when there have been very minor changes. Index: WHAT_IS_NEW.txt =================================================================== RCS file: /cvsroot/spambayes/spambayes/WHAT_IS_NEW.txt,v retrieving revision 1.15 retrieving revision 1.16 diff -C2 -d -r1.15 -r1.16 *** WHAT_IS_NEW.txt 1 Sep 2003 23:58:22 -0000 1.15 --- WHAT_IS_NEW.txt 2 Sep 2003 01:31:03 -0000 1.16 *************** *** 151,155 **** 779049, 765912, 777026, 777165, 693387, 690418, 719586, 769346, 761499, 769346, 773452, 765042, 760062, 768162, 768221, 797776, 797316, 796996, ! 797890 A url containing the details of these bugs can be made by appending the --- 151,155 ---- 779049, 765912, 777026, 777165, 693387, 690418, 719586, 769346, 761499, 769346, 773452, 765042, 760062, 768162, 768221, 797776, 797316, 796996, ! 797890, 788845 A url containing the details of these bugs can be made by appending the Index: imapfilter.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/imapfilter.py,v retrieving revision 1.52 retrieving revision 1.53 diff -C2 -d -r1.52 -r1.53 *** imapfilter.py 2 Sep 2003 01:03:16 -0000 1.52 --- imapfilter.py 2 Sep 2003 01:31:03 -0000 1.53 *************** *** 246,251 **** --- 246,263 ---- folders = [] for fol in all_folders: + # Sigh. Some servers may give us back the folder name as a + # literal, so we need to crunch this out. + if isinstance(fol, ()): + r = re.compile(r"{\d+}") + m = r.search(fol[0]) + if not m: + # Something is wrong here! Skip this folder + continue + fol = '%s"%s"' % (fol[0][:m.start()], fol[1]) r = re.compile(r"\(([\w\\ ]*)\) ") m = r.search(fol) + if not m: + # Something is not good with this folder, so skip it. + continue name_attributes = fol[:m.end()-1] # IMAP is a truly odd protocol. The delimiter is *************** *** 720,728 **** if options["globals", "verbose"]: print "Loading database %s..." % (bdbname), ! ! if useDBM: ! classifier = storage.DBDictClassifier(bdbname) ! else: ! classifier = storage.PickledClassifier(bdbname) if options["globals", "verbose"]: --- 732,737 ---- if options["globals", "verbose"]: print "Loading database %s..." % (bdbname), ! ! classifier = storage.open_storage(bdbname, useDBM) if options["globals", "verbose"]: From anadelonbrin at users.sourceforge.net Mon Sep 1 21:42:01 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon Sep 1 23:42:05 2003 Subject: [Spambayes-checkins] website download.ht,1.15,1.16 Message-ID: Update of /cvsroot/spambayes/website In directory sc8-pr-cvs1:/tmp/cvs-serv32236 Modified Files: download.ht Log Message: Fix [ 796434 ] email package link is defunct (Thanks Barry). Index: download.ht =================================================================== RCS file: /cvsroot/spambayes/website/download.ht,v retrieving revision 1.15 retrieving revision 1.16 diff -C2 -d -r1.15 -r1.16 *** download.ht 12 Aug 2003 09:57:33 -0000 1.15 --- download.ht 2 Sep 2003 03:41:55 -0000 1.16 *************** *** 12,16 ****

    Either:
  • Python 2.2.2, Python 2.3b1, or a CVS build of python, or !
  • Python 2.2, 2.2.1, plus the latest email package.

Once you've downloaded and unpacked the source archive, do the regular setup.py build; setup.py install dance, then: --- 12,16 ----

    Either:
  • Python 2.2.2, Python 2.3b1, or a CVS build of python, or !
  • Python 2.2, 2.2.1, plus the latest email package.

Once you've downloaded and unpacked the source archive, do the regular setup.py build; setup.py install dance, then: From anadelonbrin at users.sourceforge.net Tue Sep 2 00:48:58 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Tue Sep 2 02:49:02 2003 Subject: [Spambayes-checkins] spambayes/windows autoconfigure.py,NONE,1.1 Message-ID: Update of /cvsroot/spambayes/spambayes/windows In directory sc8-pr-cvs1:/tmp/cvs-serv24581/windows Added Files: autoconfigure.py Log Message: This script will read the configuration file for a mail client and changes it to use the spambayes pop3/smtp proxies (or imapfilter at some point), and sets up the bayescustomize.ini file to the correct servers/ports. Where possible, it also creates filtering rules as well. Currently works for Eudora 5.2 (including the filtering rule), Mozilla 1.3 Mail, and Opera 7.11 Mail (M2). Outlook Express is the next (much more difficult) target. --- NEW FILE: autoconfigure.py --- #!/usr/bin/env python """Automatically set up the user's mail client and SpamBayes. Currently works with: o Eudora (POP3/SMTP only) o Mozilla Mail (POP3/SMTP only) o Opera Mail (M2) (POP3/SMTP only) To do: o Establish which mail client(s) are to be setup. o Locate the appropriate configuration directory (e.g. for Eudora this is probably either the application directory, or c:\documents and settings\username\application data\qualcomm\eudora, i.e. sh_appdata\qualcomm\eudora) o This will create some unnecessary proxies in some cases. For example, if I have my client set up to get mail from pop.example.com for the user 'tmeyer' and the user 'tonym', two proxies will be created, but only one is necessary. We should check the existing proxies before adding a new one. o Figure out Outlook Express's pop3uidl.dbx file and how to hook into it (use the oe_mailbox.py module) o Other mail clients? Other platforms? o This won't work all that well if multiple mail clients are used (they will end up trying to use the same ports). In such a case, we really need to keep track of if the server is being proxied already, and reuse ports, but this is complicated. o We currently don't make any moves to protect the original file, so if something does wrong, it's corrupted. We also write into the file, rather than a temporary one and then copy across. This should all be fixed. o Suggestions? """ # This module is part of the spambayes project, which is Copyright 2002-3 # The Python Software Foundation and is covered by the Python Software # Foundation license. __author__ = "Tony Meyer " __credits__ = "All the Spambayes folk." try: True, False except NameError: # Maintain compatibility with Python 2.2 True, False = 1, 0 ## Tested with: ## o Eudora 5.2 on Windows XP ## o Mozilla 1.3 on Windows XP ## o Opera 7.11 on Windows XP import re import os import types import socket import StringIO import ConfigParser from spambayes.Options import options, optionsPathname def move_to_next_free_port(port): # Increment port until we get to one that isn't taken # I doubt this will work if there is a firewall that prevents # localhost connecting to particular ports, but I'm not sure # how else we can do this - Richie says that bind() doesn't # necessarily fail if the port is already bound. while True: try: port += 1 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect(("127.0.0.1", port)) s.close() except socket.error: return port # Let's be safe and use high ports, starting at 1110 and 1025, and going up # as required. pop_proxy_port = move_to_next_free_port(1109) smtp_proxy_port = move_to_next_free_port(1024) def configure_eudora(config_location): """Configure Eudora to use the SpamBayes POP3 and SMTP proxies, and configure SpamBayes to proxy the servers that Eudora was connecting to. """ ini_filename = "%s%seudora.ini" % (config_location, os.sep) c = ConfigParser.ConfigParser() c.read(ini_filename) translate = {("PopServer", "POPPort") : "pop3proxy", ("SMTPServer", "SMTPPort") : "smtpproxy", } pop_proxy = pop_proxy_port smtp_proxy = smtp_proxy_port for sect in c.sections(): if sect.startswith("Persona-") or sect == "Settings": if c.get(sect, "UsesIMAP") == "0": # Eudora stores the POP3 server name in two places. # Why? Who knows? We do the popaccount one # separately, because it also has the username. p = c.get(sect, "popaccount") c.set(sect, "popaccount", "%s@localhost" % \ (p[:p.index('@')],)) for ((eud_name, eud_port), us_name in translate.items(): try: port = c.get(sect, eud_port) except ConfigParser.NoOptionError: port = None if us_name.lower()[:4] == "pop3": if port is None: port = 110 pop_proxy = move_to_next_free_port(pop_proxy) proxy_port = pop_proxy else: if port is None: port = 25 smtp_proxy = move_to_next_free_port(smtp_proxy) proxy_port = smtp_proxy server = "%s:%s" % (c.get(sect, eud_name), port) options[us_name, "remote_servers"] += (server,) options[us_name, "listen_ports"] += (proxy_port,) if options["globals", "verbose"]: print "[%s] Proxy %s on localhost:%s" % \ (sect, server, proxy_port) c.set(sect, eud_name, "localhost") c.set(sect, eud_port, proxy_port) else: # Setup imapfilter instead pass out = file(ini_filename, "w") c.write(out) out.close() options.update_file(optionsPathname) # Setup filtering rule # This assumes that the spam and unsure folders already exist! # (Creating them shouldn't be that difficult - it's just a mbox file, # and I think the .toc file is automatically created). Left for # another day, however. filter_filename = "%s%sFilters.pce" % (config_location, os.sep) spam_folder_name = "Junk Mail" unsure_folder_name = "Possible Junk" header_name = options["Headers", "classification_header_name"] spam_tag = options["Headers", "header_spam_string"] unsure_tag = options["Headers", "header_unsure_string"] # We are assuming that a rules file already exists, otherwise there # is a bit more to go at the top. filter_rules = "rule SpamBayes-Spam\n" \ "transfer %s.mbx\n" \ "incoming\n" \ "header %s\n" \ "verb contains\n" \ "value %s\n" \ "conjunction ignore\n" \ "header \n" \ "verb contains\n" \ "value \n" \ "rule SpamBayes-Unsure\n" \ "transfer %s.mbx\n" \ "incoming\n" \ "header %s\n" \ "verb contains\n" \ "value %s\n" \ "conjunction ignore\n" \ "header \n" \ "verb contains\n" \ "value \n" % (spam_folder_name, header_name, spam_tag, unsure_folder_name, header_name, unsure_tag) filter_file = file(filter_filename, "a") filter_file.write(filter_rules) filter_file.close() def configure_mozilla(config_location): """Configure Mozilla to use the SpamBayes POP3 and SMTP proxies, and configure SpamBayes to proxy the servers that Mozilla was connecting to.""" prefs_file = file("%s%sprefs.js" % (config_location, os.sep), "r") prefs = prefs_file.read() prefs_file.close() save_prefs = prefs pop_accounts = {} smtp_accounts = {} r = re.compile(r"user_pref\(\"mail.server.server(\d+).(real)?hostname\", \"([^\"]*)\"\);") current_pos = 0 while True: m = r.search(prefs[current_pos:]) if not m: break server_num = m.group(1) real = m.group(2) or '' server = m.group(3) current_pos += m.end() old_pref = 'user_pref("mail.server.server%s.%shostname", "%s");' % \ (server_num, real, server) # Find the port, if there is one port_string = 'user_pref("mail.server.server%s.port", ' % \ (server_num,) port_loc = prefs.find(port_string) if port_loc == -1: port = "110" old_port = None else: loc_plus_len = port_loc + len(port_string) end_of_number = loc_plus_len + prefs[loc_plus_len:].index(')') port = prefs[loc_plus_len : end_of_number] old_port = "%s%s);" % (port_string, port) # Find the type of connection type_string = 'user_pref("mail.server.server%s.type", "' % \ (server_num,) type_loc = prefs.find(type_string) if type_loc == -1: # no type, so ignore this one continue type_loc += len(type_string) account_type = prefs[type_loc : \ type_loc + prefs[type_loc:].index('"')] if account_type == "pop3": new_pref = 'user_pref("mail.server.server%s.%shostname", ' \ '"127.0.0.1");' % (server_num, real) if not pop_accounts.has_key(server_num) or real: pop_accounts[server_num] = (new_pref, old_pref, old_port, server, port) elif account_type == "imap": # Setup imapfilter instead pass proxy_port = pop_proxy_port for num, (pref, old_pref, old_port, server, port) in pop_accounts.items(): server = "%s:%s" % (server, port) proxy_port = move_to_next_free_port(proxy_port) port_pref = 'user_pref("mail.server.server%s.port", %s);' % \ (num, proxy_port) options["pop3proxy", "remote_servers"] += (server,) options["pop3proxy", "listen_ports"] += (proxy_port,) if old_port is None: pref = "%s\n%s" % (pref, port_pref) else: save_prefs = save_prefs.replace(old_port, port_pref) save_prefs = save_prefs.replace(old_pref, pref) if options["globals", "verbose"]: print "[%s] Proxy %s on localhost:%s" % \ (num, server, proxy_port) # Do the SMTP server. # Mozilla recommends that only advanced users setup more than one, # so we'll just set that one up. Advanced users can setup SpamBayes # themselves . prefs = save_prefs r = re.compile(r"user_pref\(\"mail.smtpserver.smtp(\d+).hostname\", \"([^\"]*)\"\);") current_pos = 0 while True: m = r.search(prefs[current_pos:]) if not m: break current_pos = m.end() server_num = m.group(1) server = m.group(2) old_pref = 'user_pref("mail.smtpserver.smtp%s.hostname", ' \ '"%s");' % (server_num, server) new_pref = 'user_pref("mail.smtpserver.smtp%s.hostname", ' \ '"127.0.0.1");' % (server_num,) # Find the port port_string = 'user_pref("mail.smtpserver.smtp1.port", ' port_loc = prefs.find(port_string) if port_loc == -1: port = "25" old_port = None else: loc_plus_len = port_loc + len(port_string) end_of_number = loc_plus_len + prefs[loc_plus_len:].index(')') port = prefs[loc_plus_len : end_of_number] old_port = 'user_pref("mail.smtpserver.smtp%s.port", %s);' % \ (server_num, port) smtp_accounts[server_num] = (new_pref, old_pref, old_port, server, port) proxy_port = smtp_proxy_port for num, (pref, old_pref, old_port, server, port) in smtp_accounts.items(): server = "%s:%s" % (server, port) proxy_port = move_to_next_free_port(proxy_port) port_pref = 'user_pref("mail.smtpserver.smtp%s.port", %s);' % \ (num, proxy_port) options["smtpproxy", "remote_servers"] += (server,) options["smtpproxy", "listen_ports"] += (proxy_port,) if old_port is None: pref = "%s\n%s" % (pref, port_pref) else: save_prefs = save_prefs.replace(old_port, port_pref) save_prefs = save_prefs.replace(old_pref, pref) if options["globals", "verbose"]: print "[%s] Proxy %s on localhost:%s" % \ (num, server, proxy_port) prefs_file = file("%s%sprefs.js" % (config_location, os.sep), "w") prefs_file.write(save_prefs) prefs_file.close() options.update_file(optionsPathname) # Setup filtering rules. # Assumes that the folders already exist! I don't know how difficult # it would be to create new Mozilla mail folders. filter_filename = "%s%smsgFilterRules.dat" % (config_location, os.sep) store_name = "" # how do we get this? spam_folder_url = "mailbox:////%s//Junk%20Mail" % (store_name,) unsure_folder_url = "mailbox:////%s//Possible%20Junk" % (store_name,) header_name = options["Headers", "classification_header_name"] spam_tag = options["Headers", "header_spam_string"] unsure_tag = options["Headers", "header_unsure_string"] rule = 'name="SpamBayes-Spam"\n' \ 'enabled="yes"\n' \ 'type="1"\n' \ 'action="Move to folder"\n' \ 'actionValue="%s"\n' \ 'condition="OR (\"%s\",contains,%s)"\n' \ 'name="SpamBayes-Unsure"\n' \ 'enabled="yes"\n' \ 'type="1"\n' \ 'action="Move to folder"\n' \ 'actionValue="%s"\n' \ 'condition="OR (\"%s\",contains,%s)"\n' % \ (spam_folder_url, header_name, spam_tag, unsure_folder_url, header_name, unsure_tag) # This should now be written to the file, but I'm not sure how we # determine which subdirectory it goes into - does it have to go # into them all? # We are assuming that a rules file already exists, otherwise there # is a bit more to go at the top. def configure_m2(config_location): """Configure M2 (Opera's mailer) to use the SpamBayes POP3 and SMTP proxies, and configure SpamBayes to proxy the servers that M2 was connecting to.""" ini_filename = os.path.join(config_location, "Mail", "accounts.ini") ini_file = file(ini_filename, "r") faked_up = StringIO.StringIO() faked_up.write(";") # Missing at the start faked_up.write(ini_file.read()) faked_up.seek(0) ini_file.close() c = ConfigParser.ConfigParser() c.readfp(faked_up) translate = {("Incoming Servername", "Incoming Port") : "pop3proxy", ("Outgoing Servername", "Outgoing Port") : "smtpproxy", } pop_proxy = pop_proxy_port smtp_proxy = smtp_proxy_port for sect in c.sections(): if sect.startswith("Account") and sect != "Accounts": if c.get(sect, "Incoming Protocol") == "POP": for (m2_name, m2_port), us_name in translate.items(): try: port = c.get(sect, m2_port) except ConfigParser.NoOptionError: port = None if us_name.lower()[:4] == "pop3": if port is None: port = 110 pop_proxy = move_to_next_free_port(pop_proxy) proxy_port = pop_proxy else: if port is None: port = 25 smtp_proxy = move_to_next_free_port(smtp_proxy) proxy_port = smtp_proxy server = "%s:%s" % (c.get(sect, m2_name), port) options[us_name, "remote_servers"] += (server,) options[us_name, "listen_ports"] += (proxy_port,) if options["globals", "verbose"]: print "[%s] Proxy %s on localhost:%s" % \ (sect, server, proxy_port) c.set(sect, m2_name, "localhost") c.set(sect, m2_port, proxy_port) elif c.get(sect, "Incoming Protocol") == "IMAP": # Setup imapfilter instead pass out = file(ini_filename, "w") c.write(out) out.close() options.update_file(optionsPathname) # Setting up a filter in M2 is very simple, but I'm not sure what the # right rule is - M2 doesn't move mail, it just displays a subset. # If someone can describe the best all-purpose rule, I'll pop it in # here. if __name__ == "__main__": #configure_eudora(eudora_ini_dir) #configure_mozilla(mozilla_ini_dir) #configure_m2(m2_ini_dir) pass From anadelonbrin at users.sourceforge.net Tue Sep 2 00:52:55 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Tue Sep 2 02:52:59 2003 Subject: [Spambayes-checkins] spambayes/windows autoconfigure.py,1.1,1.2 Message-ID: Update of /cvsroot/spambayes/spambayes/windows In directory sc8-pr-cvs1:/tmp/cvs-serv25160/windows Modified Files: autoconfigure.py Log Message: Opps. Syntax error from a last minute edit. Index: autoconfigure.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/windows/autoconfigure.py,v retrieving revision 1.1 retrieving revision 1.2 diff -C2 -d -r1.1 -r1.2 *** autoconfigure.py 2 Sep 2003 06:48:55 -0000 1.1 --- autoconfigure.py 2 Sep 2003 06:52:53 -0000 1.2 *************** *** 61,65 **** def move_to_next_free_port(port): ! # Increment port until we get to one that isn't taken # I doubt this will work if there is a firewall that prevents # localhost connecting to particular ports, but I'm not sure --- 61,65 ---- def move_to_next_free_port(port): ! # Increment port until we get to one that isn't taken. # I doubt this will work if there is a firewall that prevents # localhost connecting to particular ports, but I'm not sure *************** *** 104,108 **** c.set(sect, "popaccount", "%s@localhost" % \ (p[:p.index('@')],)) ! for ((eud_name, eud_port), us_name in translate.items(): try: port = c.get(sect, eud_port) --- 104,108 ---- c.set(sect, "popaccount", "%s@localhost" % \ (p[:p.index('@')],)) ! for (eud_name, eud_port), us_name in translate.items(): try: port = c.get(sect, eud_port) *************** *** 398,401 **** --- 398,404 ---- # here. + def configure_outlook_express(config_location): + dbx_filename = os.path.join(config_location, "pop3uidl.dbx") + if __name__ == "__main__": *************** *** 403,405 **** --- 406,409 ---- #configure_mozilla(mozilla_ini_dir) #configure_m2(m2_ini_dir) + #configure_outlook_express(oe_ini_dir) pass From anadelonbrin at users.sourceforge.net Tue Sep 2 02:29:03 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Tue Sep 2 04:29:10 2003 Subject: [Spambayes-checkins] spambayes/windows autoconfigure.py,1.2,1.3 Message-ID: Update of /cvsroot/spambayes/spambayes/windows In directory sc8-pr-cvs1:/tmp/cvs-serv7322/windows Modified Files: autoconfigure.py Log Message: We can now auto-setup Outlook Express as well :) Index: autoconfigure.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/windows/autoconfigure.py,v retrieving revision 1.2 retrieving revision 1.3 diff -C2 -d -r1.2 -r1.3 *** autoconfigure.py 2 Sep 2003 06:52:53 -0000 1.2 --- autoconfigure.py 2 Sep 2003 08:29:01 -0000 1.3 *************** *** 7,10 **** --- 7,11 ---- o Mozilla Mail (POP3/SMTP only) o Opera Mail (M2) (POP3/SMTP only) + o Outlook Express (POP3/SMTP only) To do: *************** *** 47,53 **** ## Tested with: ! ## o Eudora 5.2 on Windows XP ! ## o Mozilla 1.3 on Windows XP ! ## o Opera 7.11 on Windows XP import re --- 48,55 ---- ## Tested with: ! ## o Eudora 5.2 on Windows XP ! ## o Mozilla 1.3 on Windows XP ! ## o Opera 7.11 on Windows XP ! ## o Outlook Express 6 on Windows XP import re *************** *** 398,409 **** # here. ! def configure_outlook_express(config_location): ! dbx_filename = os.path.join(config_location, "pop3uidl.dbx") if __name__ == "__main__": #configure_eudora(eudora_ini_dir) #configure_mozilla(mozilla_ini_dir) #configure_m2(m2_ini_dir) ! #configure_outlook_express(oe_ini_dir) pass --- 400,481 ---- # here. ! def configure_outlook_express(key): ! """Configure OE to use the SpamBayes POP3 and SMTP proxies, and ! configure SpamBayes to proxy the servers that OE was connecting to.""" ! # OE stores its configuration in the registry, not a file. ! ! key = key + "\\Software\\Microsoft\\Internet Account Manager\\Accounts" ! ! import win32api ! import win32con ! ! translate = {("POP3 Server", "POP3 Port") : "pop3proxy", ! ("SMTP Server", "SMTP Port") : "smtpproxy", ! } ! ! pop_proxy = pop_proxy_port ! smtp_proxy = smtp_proxy_port ! ! reg = win32api.RegOpenKeyEx(win32con.HKEY_USERS, key) ! account_index = 0 ! while True: ! # Loop through all the accounts ! config = {} ! try: ! subkey_name = "%s\\%s" % \ ! (key, win32api.RegEnumKey(reg, account_index)) ! except win32api.error: ! break ! account_index += 1 ! index = 0 ! subkey = win32api.RegOpenKeyEx(win32con.HKEY_USERS, subkey_name, 0, ! win32con.KEY_READ | win32con.KEY_SET_VALUE) ! while True: ! # Loop through all the keys ! try: ! raw = win32api.RegEnumValue(subkey, index) ! except win32api.error: ! break ! config[raw[0]] = (raw[1], raw[2]) ! index += 1 ! ! # Process this account ! if config.has_key("POP3 Server"): ! for (server_key, port_key), sect in translate.items(): ! server = "%s:%s" % (config[server_key][0], ! config[port_key][0]) ! if sect[:4] == "pop3": ! pop_proxy = move_to_next_free_port(pop_proxy) ! proxy = pop_proxy ! else: ! smtp_proxy = move_to_next_free_port(smtp_proxy) ! proxy = smtp_proxy ! options[sect, "remote_servers"] += (server,) ! options[sect, "listen_ports"] += (proxy,) ! win32api.RegSetValueEx(subkey, server_key, 0, ! win32con.REG_SZ, "127.0.0.1") ! win32api.RegSetValueEx(subkey, port_key, 0, ! win32con.REG_SZ, str(proxy)) ! if options["globals", "verbose"]: ! print "[%s] Proxy %s on localhost:%s" % \ ! (config["Account Name"][0], server, proxy) ! elif config.has_key("IMAP Server"): ! # Setup imapfilter instead. ! pass ! ! options.update_file(optionsPathname) ! ! # Outlook Express rules are done in much the same way. Should one ! # be set up to work with notate_to or notate_subject? (and set that ! # option, obviously) if __name__ == "__main__": + # XXX This is my OE key = "S-1-5-21-95318837-410984162-318601546-13224" + # XXX but I presume it's different for everyone? I'll have to check on + # XXX another machine. #configure_eudora(eudora_ini_dir) #configure_mozilla(mozilla_ini_dir) #configure_m2(m2_ini_dir) ! configure_outlook_express() pass From xenogeist at users.sourceforge.net Tue Sep 2 14:09:31 2003 From: xenogeist at users.sourceforge.net (Adam Walker) Date: Tue Sep 2 16:09:36 2003 Subject: [Spambayes-checkins] spambayes/windows/resources .cvsignore, NONE, 1.1 sb-started.ico, NONE, 1.1 sb-stopped.ico, NONE, 1.1 Message-ID: Update of /cvsroot/spambayes/spambayes/windows/resources In directory sc8-pr-cvs1:/tmp/cvs-serv30326/windows/resources Added Files: .cvsignore sb-started.ico sb-stopped.ico Log Message: icons by Romain GUY. --- NEW FILE: .cvsignore --- dialogs.aps --- NEW FILE: sb-started.ico --- (This appears to be a binary file; contents omitted.) --- NEW FILE: sb-stopped.ico --- (This appears to be a binary file; contents omitted.) From xenogeist at users.sourceforge.net Tue Sep 2 14:12:15 2003 From: xenogeist at users.sourceforge.net (Adam Walker) Date: Tue Sep 2 16:12:18 2003 Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py,1.4,1.5 Message-ID: Update of /cvsroot/spambayes/spambayes/windows In directory sc8-pr-cvs1:/tmp/cvs-serv31083/windows Modified Files: pop3proxy_tray.py Log Message: Switch icons based on if the proxy is running or not. Provide some info found on the information page in the tooltip of the icon. Index: pop3proxy_tray.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_tray.py,v retrieving revision 1.4 retrieving revision 1.5 diff -C2 -d -r1.4 -r1.5 *** pop3proxy_tray.py 2 Sep 2003 00:15:12 -0000 1.4 --- pop3proxy_tray.py 2 Sep 2003 20:12:13 -0000 1.5 *************** *** 73,90 **** # Get the custom icon ! iconPathName = "%s\\windows\\resources\\sbicon.ico" % \ (os.path.dirname(pop3proxy.__file__),) # When 1.0a6 is released, the above line will need to change to: ## iconPathName = "%s\\..\\windows\\resources\\sbicon.ico" % \ ## (os.path.dirname(pop3proxy.__file__),) ! if os.path.isfile(iconPathName): icon_flags = win32con.LR_LOADFROMFILE | win32con.LR_DEFAULTSIZE ! hicon = LoadImage(hinst, iconPathName, win32con.IMAGE_ICON, 0, ! 0, icon_flags) flags = NIF_ICON | NIF_MESSAGE | NIF_TIP ! nid = (self.hwnd, 0, flags, WM_TASKBAR_NOTIFY, hicon, "SpamBayes") Shell_NotifyIcon(NIM_ADD, nid) self.started = False # Start up pop3proxy --- 73,95 ---- # Get the custom icon ! startedIconPathName = "%s\\windows\\resources\\sb-started.ico" % \ ! (os.path.dirname(pop3proxy.__file__),) ! stoppedIconPathName = "%s\\windows\\resources\\sb-stopped.ico" % \ (os.path.dirname(pop3proxy.__file__),) # When 1.0a6 is released, the above line will need to change to: ## iconPathName = "%s\\..\\windows\\resources\\sbicon.ico" % \ ## (os.path.dirname(pop3proxy.__file__),) ! if os.path.isfile(startedIconPathName) and os.path.isfile(stoppedIconPathName): icon_flags = win32con.LR_LOADFROMFILE | win32con.LR_DEFAULTSIZE ! self.hstartedicon = LoadImage(hinst, startedIconPathName, win32con.IMAGE_ICON, 0, ! 0, icon_flags) ! self.hstoppedicon = LoadImage(hinst, stoppedIconPathName, win32con.IMAGE_ICON, 0, ! 0, icon_flags) flags = NIF_ICON | NIF_MESSAGE | NIF_TIP ! nid = (self.hwnd, 0, flags, WM_TASKBAR_NOTIFY, self.hstartedicon, "SpamBayes") Shell_NotifyIcon(NIM_ADD, nid) self.started = False + self.tip = None # Start up pop3proxy *************** *** 93,97 **** # XXX start that, and if not kick pop3proxy off in a separate thread. pop3proxy.prepare(state=pop3proxy.state) ! self.StartProxyThread() def OnDestroy(self, hwnd, msg, wparam, lparam): --- 98,124 ---- # XXX start that, and if not kick pop3proxy off in a separate thread. pop3proxy.prepare(state=pop3proxy.state) ! self.StartStop() ! ! def BuildToolTip(self): ! tip = None ! if self.started == True: ! #%i spam %i unsure %i session %i active ! tip = "SpamBayes %i spam %i ham %i unsure %i sessions %i active" %\ ! (pop3proxy.state.numSpams, pop3proxy.state.numHams, pop3proxy.state.numUnsure, ! pop3proxy.state.totalSessions, pop3proxy.state.activeSessions) ! else: ! tip = "SpamBayes is not running" ! return tip ! ! ! def UpdateIcon(self, hicon=None): ! flags = NIF_TIP ! if hicon is not None: ! flags |= NIF_ICON ! else: ! hicon = 0 ! self.tip = self.BuildToolTip() ! nid = (self.hwnd, 0, flags, WM_TASKBAR_NOTIFY, hicon, self.tip) ! Shell_NotifyIcon(NIM_MODIFY, nid) def OnDestroy(self, hwnd, msg, wparam, lparam): *************** *** 101,104 **** --- 128,134 ---- def OnTaskbarNotify(self, hwnd, msg, wparam, lparam): + if lparam==win32con.WM_MOUSEMOVE: + if self.tip != self.BuildToolTip(): + self.UpdateIcon() if lparam==win32con.WM_LBUTTONUP: # We ignore left clicks *************** *** 157,164 **** --- 187,196 ---- self.control_functions[START_STOP_ID] = ("Start SpamBayes", self.StartStop) + self.UpdateIcon(self.hstoppedicon) else: self.StartProxyThread() self.control_functions[START_STOP_ID] = ("Stop SpamBayes", self.StartStop) + self.UpdateIcon(self.hstartedicon) def OpenInterface(self): From richiehindle at users.sourceforge.net Tue Sep 2 16:47:50 2003 From: richiehindle at users.sourceforge.net (Richie Hindle) Date: Tue Sep 2 18:47:54 2003 Subject: [Spambayes-checkins] spambayes/spambayes Dibbler.py,1.6,1.7 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv31385 Modified Files: Dibbler.py Log Message: Remove gratuitous quotes from "You must log in.""" Index: Dibbler.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/Dibbler.py,v retrieving revision 1.6 retrieving revision 1.7 diff -C2 -d -r1.6 -r1.7 *** Dibbler.py 1 Sep 2003 06:07:27 -0000 1.6 --- Dibbler.py 2 Sep 2003 22:47:48 -0000 1.7 *************** *** 333,337 **** def getCancelMessage(self): """Override: Specify the cancel message for an HTTP Authentication.""" ! return "You must log in.""" --- 333,337 ---- def getCancelMessage(self): """Override: Specify the cancel message for an HTTP Authentication.""" ! return "You must log in." From anadelonbrin at users.sourceforge.net Tue Sep 2 18:37:02 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Tue Sep 2 20:37:05 2003 Subject: [Spambayes-checkins] spambayes imapfilter.py,1.53,1.54 Message-ID: Update of /cvsroot/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv15432 Modified Files: imapfilter.py Log Message: We would crash if we hadn't set anything up and didn't prompt for password. (I could swear that I've fixed this bug before...) Index: imapfilter.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/imapfilter.py,v retrieving revision 1.53 retrieving revision 1.54 diff -C2 -d -r1.53 -r1.54 *** imapfilter.py 2 Sep 2003 01:31:03 -0000 1.53 --- imapfilter.py 3 Sep 2003 00:36:59 -0000 1.54 *************** *** 753,756 **** --- 753,757 ---- pwd = pwd[0] else: + pwd = None if not launchUI: print "You need to specify both a server and a username." From kennypitt at hotmail.com Wed Sep 3 12:00:19 2003 From: kennypitt at hotmail.com (Kenny Pitt) Date: Thu Sep 4 14:13:50 2003 Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py, NONE, 1.1 In-Reply-To: Message-ID: <000001c3722c$1a46c940$300a10ac@spidynamics.com> Tony Meyer wrote: > Update of /cvsroot/spambayes/spambayes/windows > In directory sc8-pr-cvs1:/tmp/cvs-serv13593/windows > > Added Files: > pop3proxy_tray.py > Log Message: > A basic shell for a pop3proxy tray application, which can open up the > configuration page, and start and stop the proxy. The start/stop > code needs a lot more work, as part of the pop3proxy binary process. > > Note that this will cause Python 2.3 to unexpectedly terminate, but > will work fine in Python 2.2. See [ 798452 ] win32gui_taskbar.py > broken in Python 2.3 > > > --- NEW FILE: pop3proxy_tray.py --- [snip] Anyone know if something is up with the SourceForge CVS servers? I thought the time delay for us read-only types was supposed to be getting better, but I received this checkin email on Monday at 6:19am and here it is Wednesday morning and "cvs update" still hasn't pulled the first version of the file. -- Kenny Pitt From mhammond at users.sourceforge.net Thu Sep 4 06:14:14 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Thu Sep 4 17:15:26 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000 addin.py, 1.106, 1.107 filter.py, 1.31, 1.32 manager.py, 1.83, 1.84 msgstore.py, 1.71, 1.72 tester.py, 1.16, 1.17 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000 In directory sc8-pr-cvs1:/tmp/cvs-serv21153 Modified Files: addin.py filter.py manager.py msgstore.py tester.py Log Message: Start using msgstore exception objects: * msgstore no longer returns None for GetMessage/GetFolder - now always returns an object, or raises a msgstore exception. Any message operation could yield such an error (as the object may be moved after the message object has been instantiated) * addin now catches these exceptions rather than checking None * filter now catches these exceptions rather than pythoncom.com_error exceptions directly. * Support in the test suite for "forcing" exceptions. When the test suite is running, we can simulate MAPI errors at various times. Thus allows us to get far more code coverage (and IIRC, Skip has a tool to actually record code coverage - I must check that out!) * test suite *only* filters our test messages, leaving all other messages as "unprocessed". When the test suite ends, these are processed as "missed messages". * test suite hacks an event object into the filter process. This speeds up the test suite enormously (an order of magnitude at least) * manager gives a nod to the test suite. Index: addin.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/addin.py,v retrieving revision 1.106 retrieving revision 1.107 diff -C2 -d -r1.106 -r1.107 *** addin.py 1 Sep 2003 03:54:12 -0000 1.106 --- addin.py 4 Sep 2003 12:14:11 -0000 1.107 *************** *** 166,208 **** manager.LogDebug(2, "ProcessMessage starting for message '%s'" \ % msgstore_message.subject) ! if not msgstore_message.IsFilterCandidate(): ! manager.LogDebug(1, "Skipping message '%s' - we don't filter ones like that!" \ ! % msgstore_message.subject) ! return ! ! if HaveSeenMessage(msgstore_message, manager): ! # Already seen this message - user probably moving it back ! # after incorrect classification. ! # If enabled, re-train as Ham ! # otherwise just ignore. ! if manager.config.training.train_recovered_spam: ! import train ! if train.been_trained_as_spam(msgstore_message, manager.classifier_data): ! need_train = True ! else: ! prop = msgstore_message.GetField(manager.config.general.field_score_name) ! # We may not have been able to save the score - re-score now ! if prop is None: ! prop = manager.score(msgstore_message) ! # If it was not previously classified as either 'Spam' or ! # 'Unsure', then this event is unlikely to be the user ! # re-classifying (and in fact it may simply be the Outlook ! # rules moving the item. ! need_train = manager.config.filter.unsure_threshold < prop * 100 ! ! if need_train: ! TrainAsHam(msgstore_message, manager) ! else: ! subject = msgstore_message.subject ! manager.LogDebug(1, "Message '%s' was previously seen, but " \ ! "did not need to be trained as ham" % subject) ! return ! if manager.config.filter.enabled: ! import filter ! disposition = filter.filter_message(msgstore_message, manager) ! print "Message '%s' had a Spam classification of '%s'" \ ! % (msgstore_message.GetSubject(), disposition) ! else: ! print "Spam filtering is disabled - ignoring new message" manager.LogDebug(2, "ProcessMessage finished for", msgstore_message) --- 166,211 ---- manager.LogDebug(2, "ProcessMessage starting for message '%s'" \ % msgstore_message.subject) ! try: ! if not msgstore_message.IsFilterCandidate(): ! manager.LogDebug(1, "Skipping message '%s' - we don't filter ones like that!" \ ! % msgstore_message.subject) ! return ! ! if HaveSeenMessage(msgstore_message, manager): ! # Already seen this message - user probably moving it back ! # after incorrect classification. ! # If enabled, re-train as Ham ! # otherwise just ignore. ! if manager.config.training.train_recovered_spam: ! import train ! if train.been_trained_as_spam(msgstore_message, manager.classifier_data): ! need_train = True ! else: ! prop = msgstore_message.GetField(manager.config.general.field_score_name) ! # We may not have been able to save the score - re-score now ! if prop is None: ! prop = manager.score(msgstore_message) ! # If it was not previously classified as either 'Spam' or ! # 'Unsure', then this event is unlikely to be the user ! # re-classifying (and in fact it may simply be the Outlook ! # rules moving the item. ! need_train = manager.config.filter.unsure_threshold < prop * 100 ! ! if need_train: ! TrainAsHam(msgstore_message, manager) ! else: ! subject = msgstore_message.subject ! manager.LogDebug(1, "Message '%s' was previously seen, but " \ ! "did not need to be trained as ham" % subject) ! return ! if manager.config.filter.enabled: ! import filter ! disposition = filter.filter_message(msgstore_message, manager) ! print "Message '%s' had a Spam classification of '%s'" \ ! % (msgstore_message.GetSubject(), disposition) ! else: ! print "Spam filtering is disabled - ignoring new message" ! except manager.message_store.NotFoundException: ! manager.LogDebug(1, "ProcessMessage had the message moved out from underneath us") manager.LogDebug(2, "ProcessMessage finished for", msgstore_message) *************** *** 340,345 **** while 1: item = self.timer_generator.next() ! if not HaveSeenMessage(item, self.manager): ! break except StopIteration: # No items left in our generator --- 343,352 ---- while 1: item = self.timer_generator.next() ! try: ! if not HaveSeenMessage(item, self.manager): ! break ! except self.manager.message_store.NotFoundException: ! # ignore messages move underneath us ! self.manager.LogDebug(1, "The new message is skipping a message that moved underneath us") except StopIteration: # No items left in our generator *************** *** 365,371 **** # generator would miss it, so we process it synchronously. if not self.use_timer or not item.UnRead: ! msgstore_message = self.manager.message_store.GetMessage(item) ! if msgstore_message is not None: ProcessMessage(msgstore_message, self.manager) else: self._StartTimer() --- 372,383 ---- # generator would miss it, so we process it synchronously. if not self.use_timer or not item.UnRead: ! ms = self.manager.message_store ! try: ! msgstore_message = ms.GetMessage(item) ProcessMessage(msgstore_message, self.manager) + except ms.MsgStoreException, details: + print "Unexpected error fetching message" + traceback.print_exc() + print details else: self._StartTimer() *************** *** 382,385 **** --- 394,400 ---- if not self.manager.config.training.train_manual_spam: return + # XXX - Theoretically we could get "not found" exception here, + # but we have never guarded for it, and never seen it. If it does + # happen life will go on, so for now we continue to ignore it. msgstore_message = self.manager.message_store.GetMessage(item) if not msgstore_message.IsFilterCandidate(): *************** *** 539,543 **** print "Executing automated tests..." tester.test(manager) - print "Tests worked." except: traceback.print_exc() --- 554,557 ---- *************** *** 572,579 **** # Delete this item as spam. spam_folder = None spam_folder_id = self.manager.config.filter.spam_folder_id if spam_folder_id: ! spam_folder = msgstore.GetFolder(spam_folder_id) ! if not spam_folder: self.manager.ReportError("You must configure the Spam folder", "Invalid Configuration") --- 586,598 ---- # Delete this item as spam. spam_folder = None + # It is unlikely that the spam folder is not specified, as the UI + # will prevent enabling. But it could be invalid. spam_folder_id = self.manager.config.filter.spam_folder_id if spam_folder_id: ! try: ! spam_folder = msgstore.GetFolder(spam_folder_id) ! except msgstore.MsgStoreException: ! pass ! if spam_folder is None: self.manager.ReportError("You must configure the Spam folder", "Invalid Configuration") *************** *** 631,657 **** # that the source folder == dest folder - restore to # the inbox in this case. - subject = msgstore_message.GetSubject() - restore_folder = msgstore_message.GetRememberedFolder() - if restore_folder is None or \ - msgstore_message.GetFolder() == restore_folder: - print "Unable to determine source folder for message '%s' - restoring to Inbox" % (subject,) - restore_folder = inbox_folder - - # Must train before moving, else we lose the message! - print "Recovering to folder '%s' and ham training message '%s' - " % (restore_folder.name, subject), - TrainAsHam(msgstore_message, self.manager) - # Do the new message state if necessary. try: ! if new_msg_state == "Read": ! msgstore_message.SetReadState(True) ! elif new_msg_state == "Unread": ! msgstore_message.SetReadState(False) ! else: ! if new_msg_state not in ["", "None", None]: ! print "*** Bad new_msg_state value: %r" % (new_msg_state,) ! except pythoncom.com_error: ! print "*** Failed to set the message state to '%s' for message '%s'" % (new_msg_state, subject) ! # Now move it. ! msgstore_message.MoveToReportingError(self.manager, restore_folder) # Note the move will possibly also trigger a re-train # but we are smart enough to know we have already done it. --- 650,681 ---- # that the source folder == dest folder - restore to # the inbox in this case. try: ! subject = msgstore_message.GetSubject() ! restore_folder = msgstore_message.GetRememberedFolder() ! if restore_folder is None or \ ! msgstore_message.GetFolder() == restore_folder: ! print "Unable to determine source folder for message '%s' - restoring to Inbox" % (subject,) ! restore_folder = inbox_folder ! ! # Must train before moving, else we lose the message! ! print "Recovering to folder '%s' and ham training message '%s' - " % (restore_folder.name, subject), ! TrainAsHam(msgstore_message, self.manager) ! # Do the new message state if necessary. ! try: ! if new_msg_state == "Read": ! msgstore_message.SetReadState(True) ! elif new_msg_state == "Unread": ! msgstore_message.SetReadState(False) ! else: ! if new_msg_state not in ["", "None", None]: ! print "*** Bad new_msg_state value: %r" % (new_msg_state,) ! except msgstore.MsgStoreException, details: ! print "*** Failed to set the message state to '%s' for message '%s'" % (new_msg_state, subject) ! print details ! # Now move it. ! msgstore_message.MoveToReportingError(self.manager, restore_folder) ! except msgstore.NotFoundException: ! # Message moved under us - ignore. ! self.manager.LogDebug(1, "Recover from spam had message moved from underneath us - ignored") # Note the move will possibly also trigger a re-train # but we are smart enough to know we have already done it. *************** *** 930,938 **** ret = [] for i in range(sel.Count): item = sel.Item(i+1) ! msgstore_message = self.manager.message_store.GetMessage(item) ! if msgstore_message and msgstore_message.IsFilterCandidate(): ! ret.append(msgstore_message) if len(ret) == 0: --- 954,970 ---- ret = [] + ms = self.manager.message_store for i in range(sel.Count): item = sel.Item(i+1) ! try: ! msgstore_message = ms.GetMessage(item) ! if msgstore_message.IsFilterCandidate(): ! ret.append(msgstore_message) ! except ms.NotFoundException: ! pass ! except ms.MsgStoreException, details: ! print "Unexpected error fetching message" ! traceback.print_exc() ! print details if len(ret) == 0: Index: filter.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/filter.py,v retrieving revision 1.31 retrieving revision 1.32 diff -C2 -d -r1.31 -r1.32 *** filter.py 25 Aug 2003 01:33:11 -0000 1.31 --- filter.py 4 Sep 2003 12:14:12 -0000 1.32 *************** *** 10,15 **** True, False = 1, 0 - import pythoncom # for the exceptions. - def filter_message(msg, mgr, all_actions=True): config = mgr.config.filter --- 10,13 ---- *************** *** 29,32 **** --- 27,31 ---- attr_prefix = None + ms = mgr.message_store try: try: *************** *** 43,59 **** msg.RememberMessageCurrentFolder() msg.Save() ! except pythoncom.com_error, (hr, exc_msg, exc, arg_err): ! # This seems to happen for IMAP mails (0x800cccd3) ! # and also for hotmail messages (0x8004dff7) ! known_failure_codes = -2146644781, -2147164169 ! # I also heard a rumour hotmail works if we do 2 saves ! if hr not in known_failure_codes: ! print "Unexpected MAPI error saving the spam score for", msg ! print hr, exc_msg, exc ! else: ! # So we can see if it still happens :) ! mgr.LogDebug(1, "Note: known (but still not understood) " \ ! "error 0x%x saving the spam score." % hr) ! # No need for a traceback in this case. # Clear dirty flag anyway msg.dirty = False --- 42,54 ---- msg.RememberMessageCurrentFolder() msg.Save() ! except ms.ReadOnlyException: ! # read-only message - not much we can do! ! # Clear dirty flag anyway ! mgr.LogDebug(1, "Message is read-only - could not save Spam score") ! msg.dirty = False ! except ms.MsgStoreException, details: ! # Some other error saving - this is nasty. ! print "Unexpected MAPI error saving the spam score for", msg ! print details # Clear dirty flag anyway msg.dirty = False *************** *** 68,73 **** pass elif action.startswith("co"): # copied ! dest_folder = mgr.message_store.GetFolder(folder_id) ! if dest_folder is None: print "ERROR: Unable to open the folder to Copy the " \ "message - this message was not copied" --- 63,69 ---- pass elif action.startswith("co"): # copied ! try: ! dest_folder = ms.GetFolder(folder_id) ! except ms.MsgStoreException: print "ERROR: Unable to open the folder to Copy the " \ "message - this message was not copied" *************** *** 75,80 **** msg.CopyToReportingError(mgr, dest_folder) elif action.startswith("mo"): # Moved ! dest_folder = mgr.message_store.GetFolder(folder_id) ! if dest_folder is None: print "ERROR: Unable to open the folder to Move the " \ "message - this message was not moved" --- 71,77 ---- msg.CopyToReportingError(mgr, dest_folder) elif action.startswith("mo"): # Moved ! try: ! dest_folder = ms.GetFolder(folder_id) ! except ms.MsgStoreException: print "ERROR: Unable to open the folder to Move the " \ "message - this message was not moved" Index: manager.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/manager.py,v retrieving revision 1.83 retrieving revision 1.84 diff -C2 -d -r1.83 -r1.84 *** manager.py 29 Aug 2003 00:10:33 -0000 1.83 --- manager.py 4 Sep 2003 12:14:12 -0000 1.84 *************** *** 329,332 **** --- 329,333 ---- self.outlook = outlook self.dialog_parser = None + self.test_suite_running = False import_early_core_spambayes_stuff() *************** *** 411,416 **** --- 412,425 ---- def ReportError(self, message, title = None): + if self.test_suite_running: + print "ReportError:", repr(message) + print "(but test suite running - not reported)" + return ReportError(message, title) def ReportInformation(self, message, title=None): + if self.test_suite_running: + print "ReportInformation:", repr(message) + print "(but test suite running - not reported)" + return ReportInformation(message, title) def AskQuestion(self, message, title=None): *************** *** 443,446 **** --- 452,459 ---- if key is None: key = msg # Always print the message and traceback. + if self.test_suite_running: + print "ReportErrorOnce:", repr(msg) + print "(but test suite running - not reported)" + return print "ERROR:", repr(msg) traceback.print_exc() *************** *** 505,513 **** names = [] for eid in folder_ids: ! folder = self.message_store.GetFolder(eid) ! if folder is None: ! name = "" ! else: name = folder.name names.append(name) ret = '; '.join(names) --- 518,526 ---- names = [] for eid in folder_ids: ! try: ! folder = self.message_store.GetFolder(eid) name = folder.name + except self.message_store.MsgStoreException: + name = "" names.append(name) ret = '; '.join(names) *************** *** 530,535 **** # (which really is OK!) assert self.outlook is not None, "I need outlook :(" ! msgstore_folder = self.message_store.GetFolder(folder_id) ! if msgstore_folder is None: print "Checking a folder for our field failed - "\ "there is no such folder." --- 543,549 ---- # (which really is OK!) assert self.outlook is not None, "I need outlook :(" ! try: ! msgstore_folder = self.message_store.GetFolder(folder_id) ! except self.message_store.MsgStoreException: print "Checking a folder for our field failed - "\ "there is no such folder." Index: msgstore.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/msgstore.py,v retrieving revision 1.71 retrieving revision 1.72 diff -C2 -d -r1.71 -r1.72 *** msgstore.py 1 Sep 2003 07:29:48 -0000 1.71 --- msgstore.py 4 Sep 2003 12:14:12 -0000 1.72 *************** *** 10,94 **** True, False = 1, 0 ! # Nod to our automated test suite - we *do* want these messages filtered! ! test_suite_running = False ! ! # Abstract definition - can be moved out when we have more than one sub-class ! # External interface to this module is almost exclusively via a "folder ID" ! ! class MsgStoreException(Exception): ! pass ! ! class NotFoundException(MsgStoreException): ! pass ! ! class MsgStore: ! # Stash exceptions in the class for ease of use by consumers. ! MsgStoreException = MsgStoreException ! NotFoundException = NotFoundException ! def __init__(self): ! pass ! def Close(self): ! # Close this object and free everything ! raise NotImplementedError ! def GetFolderGenerator(self, folder_ids, include_sub): ! # Return a generator of MsgStoreFolder objects. ! raise NotImplementedError ! def GetFolder(self, folder_id): ! # Return a single folder given the ID. ! raise NotImplementedError ! def GetMessage(self, message_id): ! # Return a single message given the ID. ! raise NotImplementedError ! ! class MsgStoreFolder: ! def __init__(self): ! self.name = "" ! self.count = 0 ! def GetParent(self): ! # return a folder object with the parent, or None ! raise NotImplementedError ! def GetMessageGenerator(self, folder, only_filter_candidates = True): ! # Return a generator of MsgStoreMsg objects for the folder ! raise NotImplementedError ! ! class MsgStoreMsg: ! def __init__(self): ! pass ! def GetEmailPackageObject(self): ! # Return a "read-only" Python email package object ! # "read-only" in that changes will never be reflected to the real store. ! raise NotImplementedError ! def SetField(self, name, value): ! # Abstractly set a user field name/id to a field value. ! # User field is for the user to see - status/internal fields ! # should get their own methods ! raise NotImplementedError ! def GetSubject(self): ! # Get the subject - function as it may require a trip to the store! ! raise NotImplementedError ! def GetField(self, name): ! # Abstractly get a user field name/id to a field value. ! raise NotImplementedError ! def Save(self): ! # Save changes after field changes. ! raise NotImplementedError ! def MoveTo(self, folder_id): ! # Move the message to a folder. ! raise NotImplementedError ! def CopyTo(self, folder_id): ! # Copy the message to a folder. ! raise NotImplementedError ! def IsFilterCandidate(self): ! # Return True if this is a message that should be checked for spam ! # Return False if it should be ignored (eg, user-composed message, ! # undeliverable report, meeting request etc. ! raise NotImplementedError ! ! # Our MAPI implementation ! import warnings ! if sys.version_info >= (2, 3): ! # sick off the new hex() warnings! ! warnings.filterwarnings("ignore", category=FutureWarning, append=1) ! from win32com.client import Dispatch, constants from win32com.mapi import mapi, mapiutil --- 10,14 ---- True, False = 1, 0 ! # MAPI imports etc. from win32com.client import Dispatch, constants from win32com.mapi import mapi, mapiutil *************** *** 96,99 **** --- 16,20 ---- import pythoncom + # Additional MAPI constants we dont have in Python MESSAGE_MOVE = 0x1 # from MAPIdefs.h MSGFLAG_READ = 0x1 # from MAPIdefs.h *************** *** 110,113 **** --- 31,96 ---- USE_DEFERRED_ERRORS = mapi.MAPI_DEFERRED_ERRORS # or set to zero to see what changes + #import warnings + #if sys.version_info >= (2, 3): + # # sick off the new hex() warnings! + # warnings.filterwarnings("ignore", category=FutureWarning, append=1) + + # Nod to our automated test suite. Currently supports a hack so our test + # message is filtered, and also for raising exceptions at key times. + # see tester.py for more details. + test_suite_running = False + test_suite_failure_request = None + test_suite_failure = None + # Sometimes the test suite will request that we simulate MAPI errors. + def help_test_suite(checkpoint_name): + if test_suite_running and \ + test_suite_failure_request == checkpoint_name: + raise test_suite_failure[0], test_suite_failure[1] + + # Exceptions raised by this module. Raw MAPI exceptions should never + # be raised to the caller. + class MsgStoreException(Exception): + def __init__(self, mapi_exception, extra_msg = None): + self.mapi_exception = mapi_exception + self.extra_msg = extra_msg + Exception.__init__(self, mapi_exception, extra_msg) + def __str__(self): + try: + return "%s: %s" % (self.__class__.__name__, + GetCOMExceptionString(self.mapi_exception)) + except: + print "Error __str__" + import traceback + traceback.print_exc() + + # Exception raised when you attempt to get a message or folder that doesn't + # exist. Usually means you are querying an ID that *was* valid, but has + # since been moved or deleted. + # Note you may get this exception "getting" objects (such as messages or + # folders), or accessing properties once the object was created (the message + # may be moved under us at any time) + class NotFoundException(MsgStoreException): + pass + + # Exception raised when you try and modify a "read only" object. + # Only currently examples are Hotmail and IMAP folders. + class ReadOnlyException(MsgStoreException): + pass + + # Utility functions for exceptions. Convert a COM exception to the best + # manager exception. + def MsgStoreExceptionFromCOMException(com_exc): + if IsNotFoundCOMException(com_exc): + return NotFoundException(com_exc) + if IsReadOnlyCOMException(com_exc): + return ReadOnlyException(com_exc) + return MsgStoreException(com_exc) + + # Build a reasonable string from a COM exception tuple + def GetCOMExceptionString(exc_val): + hr, msg, exc, arg_err = exc_val + err_string = mapiutil.GetScodeString(hr) + return "Exception 0x%x (%s): %s" % (hr, err_string, msg) + # Does this exception probably mean "object not found"? def IsNotFoundCOMException(exc_val): *************** *** 121,130 **** return hr == mapi.MAPI_E_FAILONEPROVIDER ! def GetCOMExceptionString(exc_val): ! hr, msg, exc, arg_err = exc_val ! err_string = mapiutil.GetScodeString(hr) ! return "Exception 0x%x (%s): %s" % (hr, err_string, msg) ! def ReportMAPIError(manager, what, exc_type, exc_val): hr, exc_msg, exc, arg_err = exc_val if hr == mapi.MAPI_E_TABLE_TOO_BIG: --- 104,115 ---- return hr == mapi.MAPI_E_FAILONEPROVIDER ! def IsReadOnlyCOMException(exc_val): ! # This seems to happen for IMAP mails (0x800cccd3) ! # and also for hotmail messages (0x8004dff7) ! known_failure_codes = -2146644781, -2147164169 ! return exc_val[0] in known_failure_codes ! ! def ReportMAPIError(manager, what, exc_val): hr, exc_msg, exc, arg_err = exc_val if hr == mapi.MAPI_E_TABLE_TOO_BIG: *************** *** 142,146 **** manager.ReportErrorOnce(err_msg) ! class MAPIMsgStore(MsgStore): def __init__(self, outlook = None): self.outlook = outlook --- 127,137 ---- manager.ReportErrorOnce(err_msg) ! # Our objects. ! class MAPIMsgStore: ! # Stash exceptions in the class for ease of use by consumers. ! MsgStoreException = MsgStoreException ! NotFoundException = NotFoundException ! ReadOnlyException = ReadOnlyException ! def __init__(self, outlook = None): self.outlook = outlook *************** *** 298,355 **** def GetFolder(self, folder_id): # Return a single folder given the ID. ! try: ! # See if this is an Outlook folder item ! sid = mapi.BinFromHex(folder_id.StoreID) ! eid = mapi.BinFromHex(folder_id.EntryID) ! folder_id = sid, eid ! except AttributeError: ! # No 'EntryID'/'StoreID' properties - a 'normal' ID ! folder_id = self.NormalizeID(folder_id) ! except pythoncom.com_error, details: ! if IsNotFoundCOMException(details): ! print "Unable to open folder '%r'" \ ! "- the folder was not found" % folder_id ! else: ! print "Unexpected MAPI error opening folder" ! print GetCOMExceptionString(details) ! return None ! # Also catch COM exceptions opening the folder and table. ! try: folder = self._OpenEntry(folder_id) table = folder.GetContentsTable(0) ! except pythoncom.com_error, details: ! # We will ignore *all* such errors for the time ! # being, but warn for results we don't know about. ! if not IsNotFoundCOMException(details): ! print "WARNING: Unexpected MAPI error opening folder" ! print GetCOMExceptionString(details) ! return None ! # Ensure we have a long-term ID. ! rc, props = folder.GetProps( (PR_ENTRYID, PR_DISPLAY_NAME_A), 0) ! folder_id = folder_id[0], props[0][1] ! return MAPIMsgStoreFolder(self, folder_id, props[1][1], table.GetRowCount(0)) def GetMessage(self, message_id): # Return a single message given either the ID, or an Outlook # message representing the object. ! try: ! eid = mapi.BinFromHex(message_id.EntryID) ! sid = mapi.BinFromHex(message_id.Parent.StoreID) ! message_id = sid, eid ! except AttributeError: ! # No 'EntryID'/'StoreID' properties - a 'normal' ID ! message_id = self.NormalizeID(message_id) except pythoncom.com_error, details: ! if IsNotFoundCOMException(details): ! print "Unable to open message '%r'" \ ! "- the message was not found" % message_id ! else: ! print "Unexpected MAPI error opening message" ! print GetCOMExceptionString(details) ! return None ! mapi_object = self._OpenEntry(message_id) ! hr, data = mapi_object.GetProps(MAPIMsgStoreMsg.message_init_props,0) ! return MAPIMsgStoreMsg(self, data) def YieldReceiveFolders(self, msg_class = "IPM.Note"): --- 289,327 ---- def GetFolder(self, folder_id): # Return a single folder given the ID. ! try: # catch all MAPI errors ! try: ! # See if this is an Outlook folder item ! sid = mapi.BinFromHex(folder_id.StoreID) ! eid = mapi.BinFromHex(folder_id.EntryID) ! folder_id = sid, eid ! except AttributeError: ! # No 'EntryID'/'StoreID' properties - a 'normal' ID ! folder_id = self.NormalizeID(folder_id) folder = self._OpenEntry(folder_id) table = folder.GetContentsTable(0) ! # Ensure we have a long-term ID. ! rc, props = folder.GetProps( (PR_ENTRYID, PR_DISPLAY_NAME_A), 0) ! folder_id = folder_id[0], props[0][1] ! return MAPIMsgStoreFolder(self, folder_id, props[1][1], table.GetRowCount(0)) + except pythoncom.com_error, details: + raise MsgStoreExceptionFromCOMException(details) def GetMessage(self, message_id): # Return a single message given either the ID, or an Outlook # message representing the object. ! try: # catch all MAPI exceptions. ! try: ! eid = mapi.BinFromHex(message_id.EntryID) ! sid = mapi.BinFromHex(message_id.Parent.StoreID) ! message_id = sid, eid ! except AttributeError: ! # No 'EntryID'/'StoreID' properties - a 'normal' ID ! message_id = self.NormalizeID(message_id) ! mapi_object = self._OpenEntry(message_id) ! hr, data = mapi_object.GetProps(MAPIMsgStoreMsg.message_init_props,0) ! return MAPIMsgStoreMsg(self, data) except pythoncom.com_error, details: ! raise MsgStoreExceptionFromCOMException(details) def YieldReceiveFolders(self, msg_class = "IPM.Note"): *************** *** 451,455 **** return html or '' ! class MAPIMsgStoreFolder(MsgStoreMsg): def __init__(self, msgstore, id, name, count): self.msgstore = msgstore --- 423,427 ---- return html or '' ! class MAPIMsgStoreFolder: def __init__(self, msgstore, id, name, count): self.msgstore = msgstore *************** *** 558,561 **** --- 530,535 ---- # Our restriction helped, but may not have filtered # every message we don't want to touch. + # Note no exception will be raised below if the message is + # moved under us, as we don't need to access any properties. msg = MAPIMsgStoreMsg(self.msgstore, row) if not only_filter_candidates or msg.IsFilterCandidate(): *************** *** 596,599 **** --- 570,575 ---- break for row in rows: + # Note no exception will be raised below if the message is + # moved under us, as we don't need to access any properties. msg = MAPIMsgStoreMsg(self.msgstore, row) if msg.IsFilterCandidate(): *************** *** 615,620 **** return self._FolderFromMAPIFolder(ret) ! ! class MAPIMsgStoreMsg(MsgStoreMsg): # All the properties we must initialize a message with. # These include all the IDs we need, parent IDs, any properties needed --- 591,595 ---- return self._FolderFromMAPIFolder(ret) ! class MAPIMsgStoreMsg: # All the properties we must initialize a message with. # These include all the IDs we need, parent IDs, any properties needed *************** *** 713,716 **** --- 688,694 ---- # it manages given it is an external server, and as far as I can tell, # this does not appear in the headers. + if test_suite_running: + # While the test suite is running, we *only* filter test msgs. + return self.subject == "SpamBayes addin auto-generated test message" class_check = self.msgclass.lower() for check in "ipm.note", "ipm.anti-virus": *************** *** 721,725 **** return False # Must match msg class to get here. ! return self.was_received or test_suite_running def _GetPotentiallyLargeStringProp(self, prop_id, row): --- 699,703 ---- return False # Must match msg class to get here. ! return self.was_received def _GetPotentiallyLargeStringProp(self, prop_id, row): *************** *** 848,852 **** def _EnsureObject(self): if self.mapi_object is None: ! self.mapi_object = self.msgstore._OpenEntry(self.id) def GetEmailPackageObject(self, strip_mime_headers=True): --- 826,834 ---- def _EnsureObject(self): if self.mapi_object is None: ! try: ! help_test_suite("MAPIMsgStoreMsg._EnsureObject") ! self.mapi_object = self.msgstore._OpenEntry(self.id) ! except pythoncom.com_error, details: ! raise MsgStoreExceptionFromCOMException(details) def GetEmailPackageObject(self, strip_mime_headers=True): *************** *** 951,977 **** # XXX Outlook to define a custom Integer field of the same name. self._EnsureObject() - if type(prop) != type(0): - props = ( (mapi.PS_PUBLIC_STRINGS, prop), ) - propIds = self.mapi_object.GetIDsFromNames(props, mapi.MAPI_CREATE) - type_tag = _MapiTypeMap.get(type(val)) - if type_tag is None: - raise ValueError, "Don't know what to do with '%r' ('%s')" % ( - val, type(val)) - prop = PROP_TAG(type_tag, PROP_ID(propIds[0])) - if val is None: - # Delete the property - self.mapi_object.DeleteProps((prop,)) - else: - self.mapi_object.SetProps(((prop,val),)) - self.dirty = True - - def GetField(self, prop, raise_errors = False): try: ! self._EnsureObject() except pythoncom.com_error, details: ! if not IsNotFoundCOMException(details): ! print "ERROR: Could not open an object to fetch a field" ! print details ! return None if type(prop) != type(0): props = ( (mapi.PS_PUBLIC_STRINGS, prop), ) --- 933,958 ---- # XXX Outlook to define a custom Integer field of the same name. self._EnsureObject() try: ! if type(prop) != type(0): ! props = ( (mapi.PS_PUBLIC_STRINGS, prop), ) ! propIds = self.mapi_object.GetIDsFromNames(props, mapi.MAPI_CREATE) ! type_tag = _MapiTypeMap.get(type(val)) ! if type_tag is None: ! raise ValueError, "Don't know what to do with '%r' ('%s')" % ( ! val, type(val)) ! prop = PROP_TAG(type_tag, PROP_ID(propIds[0])) ! help_test_suite("MAPIMsgStoreMsg.SetField") ! if val is None: ! # Delete the property ! self.mapi_object.DeleteProps((prop,)) ! else: ! self.mapi_object.SetProps(((prop,val),)) ! self.dirty = True except pythoncom.com_error, details: ! raise MsgStoreExceptionFromCOMException(details) ! ! def GetField(self, prop): ! # xxx - still raise_errors? ! self._EnsureObject() if type(prop) != type(0): props = ( (mapi.PS_PUBLIC_STRINGS, prop), ) *************** *** 989,996 **** return None return val ! except: ! if raise_errors: ! raise ! return None def GetReadState(self): --- 970,975 ---- return None return val ! except pythoncom.com_error, details: ! raise MsgStoreExceptionFromCOMException(details) def GetReadState(self): *************** *** 999,1046 **** def SetReadState(self, is_read): ! self._EnsureObject() ! # always try and clear any pending delivery reports of read/unread ! if is_read: ! self.mapi_object.SetReadFlag(USE_DEFERRED_ERRORS|SUPPRESS_RECEIPT) ! else: ! self.mapi_object.SetReadFlag(USE_DEFERRED_ERRORS|CLEAR_READ_FLAG) ! if __debug__: ! if self.GetReadState() != is_read: ! print "MAPI SetReadState appears to have failed to change the message state" ! print "Requested set to %s but the MAPI field after was %r" % \ ! (is_read, self.GetField(PR_MESSAGE_FLAGS)) def Save(self): assert self.dirty, "asking me to save a clean message!" ! # There are some known exceptions that can be raised by IMAP and hotmail ! # For now, we just let the caller handle all errors, and manually ! # reset the dirty flag. Only current caller is filter.py ! # There are also some issues with the "unread flag" that fiddling this ! # save code may fix. ! # It seems that *not* specifying mapi.MAPI_DEFERRED_ERRORS solves alot ! # of said problems though! So we don't! ! self.mapi_object.SaveChanges(mapi.KEEP_OPEN_READWRITE) ! self.dirty = False def _DoCopyMove(self, folder, isMove): assert not self.dirty, \ "asking me to move a dirty message - later saves will fail!" ! dest_folder = self.msgstore._OpenEntry(folder.id) ! source_folder = self.msgstore._OpenEntry(self.folder_id) ! flags = 0 ! if isMove: flags |= MESSAGE_MOVE ! eid = self.id[1] ! source_folder.CopyMessages((eid,), ! None, ! dest_folder, ! 0, ! None, ! flags) ! # At this stage, I think we have lost meaningful ID etc values ! # Set everything to None to make it clearer what is wrong should ! # this become an issue. We would need to re-fetch the eid of ! # the item, and set the store_id to the dest folder. ! self.id = None ! self.folder_id = None def MoveTo(self, folder): --- 978,1032 ---- def SetReadState(self, is_read): ! try: ! self._EnsureObject() ! # always try and clear any pending delivery reports of read/unread ! help_test_suite("MAPIMsgStoreMsg.SetReadState") ! if is_read: ! self.mapi_object.SetReadFlag(USE_DEFERRED_ERRORS|SUPPRESS_RECEIPT) ! else: ! self.mapi_object.SetReadFlag(USE_DEFERRED_ERRORS|CLEAR_READ_FLAG) ! if __debug__: ! if self.GetReadState() != is_read: ! print "MAPI SetReadState appears to have failed to change the message state" ! print "Requested set to %s but the MAPI field after was %r" % \ ! (is_read, self.GetField(PR_MESSAGE_FLAGS)) ! except pythoncom.com_error, details: ! raise MsgStoreExceptionFromCOMException(details) def Save(self): assert self.dirty, "asking me to save a clean message!" ! # It seems that *not* specifying mapi.MAPI_DEFERRED_ERRORS solves a lot ! # problems! So we don't! ! try: ! help_test_suite("MAPIMsgStoreMsg.Save") ! self.mapi_object.SaveChanges(mapi.KEEP_OPEN_READWRITE) ! self.dirty = False ! except pythoncom.com_error, details: ! raise MsgStoreExceptionFromCOMException(details) def _DoCopyMove(self, folder, isMove): assert not self.dirty, \ "asking me to move a dirty message - later saves will fail!" ! try: ! dest_folder = self.msgstore._OpenEntry(folder.id) ! source_folder = self.msgstore._OpenEntry(self.folder_id) ! flags = 0 ! if isMove: flags |= MESSAGE_MOVE ! eid = self.id[1] ! help_test_suite("MAPIMsgStoreMsg._DoCopyMove") ! source_folder.CopyMessages((eid,), ! None, ! dest_folder, ! 0, ! None, ! flags) ! # At this stage, I think we have lost meaningful ID etc values ! # Set everything to None to make it clearer what is wrong should ! # this become an issue. We would need to re-fetch the eid of ! # the item, and set the store_id to the dest folder. ! self.id = None ! self.folder_id = None ! except pythoncom.com_error, details: ! raise MsgStoreExceptionFromCOMException(details) def MoveTo(self, folder): *************** *** 1053,1068 **** # user. Any errors are re-raised so the caller can degrade gracefully if # necessary. def MoveToReportingError(self, manager, folder): try: self.MoveTo(folder) ! except pythoncom.com_error, details: ! ReportMAPIError(manager, "Moving a message", pythoncom.com_error, details) ! raise def CopyToReportingError(self, manager, folder): try: self.MoveTo(folder) ! except pythoncom.com_error, details: ! ReportMAPIError(manager, "Copying a message", pythoncom.com_error, details) ! raise def GetFolder(self): --- 1039,1055 ---- # user. Any errors are re-raised so the caller can degrade gracefully if # necessary. + # XXX - not too happy with these - they should go, and the caller should + # handle (especially now that we work exclusively with exceptions from + # this module. def MoveToReportingError(self, manager, folder): try: self.MoveTo(folder) ! except MsgStoreException, details: ! ReportMAPIError(manager, "Moving a message", details.mapi_exception) def CopyToReportingError(self, manager, folder): try: self.MoveTo(folder) ! except MsgStoreException, details: ! ReportMAPIError(manager, "Copying a message", details.mapi_exception) def GetFolder(self): *************** *** 1074,1088 **** def RememberMessageCurrentFolder(self): self._EnsureObject() ! folder = self.GetFolder() ! props = ( (mapi.PS_PUBLIC_STRINGS, "SpamBayesOriginalFolderStoreID"), ! (mapi.PS_PUBLIC_STRINGS, "SpamBayesOriginalFolderID") ! ) ! resolve_ids = self.mapi_object.GetIDsFromNames(props, mapi.MAPI_CREATE) ! prop_ids = PROP_TAG( PT_BINARY, PROP_ID(resolve_ids[0])), \ ! PROP_TAG( PT_BINARY, PROP_ID(resolve_ids[1])) ! ! prop_tuples = (prop_ids[0],folder.id[0]), (prop_ids[1],folder.id[1]) ! self.mapi_object.SetProps(prop_tuples) ! self.dirty = True def GetRememberedFolder(self): --- 1061,1078 ---- def RememberMessageCurrentFolder(self): self._EnsureObject() ! try: ! folder = self.GetFolder() ! props = ( (mapi.PS_PUBLIC_STRINGS, "SpamBayesOriginalFolderStoreID"), ! (mapi.PS_PUBLIC_STRINGS, "SpamBayesOriginalFolderID") ! ) ! resolve_ids = self.mapi_object.GetIDsFromNames(props, mapi.MAPI_CREATE) ! prop_ids = PROP_TAG( PT_BINARY, PROP_ID(resolve_ids[0])), \ ! PROP_TAG( PT_BINARY, PROP_ID(resolve_ids[1])) ! ! prop_tuples = (prop_ids[0],folder.id[0]), (prop_ids[1],folder.id[1]) ! self.mapi_object.SetProps(prop_tuples) ! self.dirty = True ! except pythoncom.com_error, details: ! raise MsgStoreExceptionFromCOMException(details) def GetRememberedFolder(self): *************** *** 1100,1103 **** --- 1090,1094 ---- (store_tag, store_id), (eid_tag, eid) = data folder_id = mapi.HexFromBin(store_id), mapi.HexFromBin(eid) + help_test_suite("MAPIMsgStoreMsg.GetRememberedFolder") return self.msgstore.GetFolder(folder_id) except: Index: tester.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/tester.py,v retrieving revision 1.16 retrieving revision 1.17 diff -C2 -d -r1.16 -r1.17 *** tester.py 1 Sep 2003 05:35:58 -0000 1.16 --- tester.py 4 Sep 2003 12:14:12 -0000 1.17 *************** *** 16,19 **** --- 16,23 ---- import rfc822 import cStringIO + import threading + + from win32com.mapi import mapi, mapiutil + import pythoncom HAM="ham" *************** *** 29,37 **** --- 33,46 ---- raise TestFailure(msg) + filter_event = threading.Event() + def WaitForFilters(): import pythoncom # Must wait longer than normal, so when run with a timer we still work. + filter_event.clear() for i in range(500): pythoncom.PumpWaitingMessages() + if filter_event.isSet(): + break sleep(0.01) *************** *** 371,374 **** --- 380,384 ---- def run_tests(manager): + "Filtering tests" driver = Driver(manager) manager.Save() # necessary after a full retrain *************** *** 384,408 **** def run_filter_tests(manager): # setup config to save info with the message, and test ! print "*" * 10, "Running tests with save_spam_info=True, timer off" ! manager.config.experimental.timer_start_delay = 0 ! manager.config.experimental.timer_interval = 0 ! manager.config.filter.save_spam_info = True ! manager.addin.FiltersChanged() # to ensure correct filtler in place ! run_tests(manager) ! # do it again with the same config, just to prove we can. ! print "*" * 10, "Running them again with save_spam_info=True" ! run_tests(manager) ! # enable the timer. ! manager.config.experimental.timer_start_delay = 1000 ! manager.config.experimental.timer_interval = 500 ! manager.addin.FiltersChanged() # to switch to timer based filters. ! print "*" * 10, "Running them again with save_spam_info=True, and timer enabled" ! run_tests(manager) ! # and with save_spam_info False. ! print "*" * 10, "Running tests with save_spam_info=False" ! manager.config.filter.save_spam_info = False ! run_tests(manager) ! print "*" * 10, "Filtering tests completed successfully." def run_nonfilter_tests(manager): # And now some other 'sanity' checks. --- 394,441 ---- def run_filter_tests(manager): # setup config to save info with the message, and test ! apply_with_new_config(manager, ! {"Filter.timer_enabled": False, ! "Filter.save_spam_info" : True, ! }, ! run_tests, manager) ! ! apply_with_new_config(manager, ! {"Filter.timer_enabled": True, ! "Filter.save_spam_info" : True, ! }, ! run_tests, manager) ! apply_with_new_config(manager, ! {"Filter.timer_enabled": False, ! "Filter.save_spam_info" : False, ! }, ! run_tests, manager) ! ! apply_with_new_config(manager, ! {"Filter.timer_enabled": True, ! "Filter.save_spam_info" : False, ! }, ! run_tests, manager) + def apply_with_new_config(manager, new_config_dict, func, *args): + old_config = {} + friendly_opts = [] + for name, val in new_config_dict.items(): + sect_name, opt_name = name.split(".") + old_config[sect_name, opt_name] = manager.options.get(sect_name, opt_name) + manager.options.set(sect_name, opt_name, val) + friendly_opts.append("%s=%s" % (name, val)) + manager.addin.FiltersChanged() # to ensure correct filtler in place + try: + test_name = getattr(func, "__doc__", None) + if not test_name: test_name = func.__name__ + print "*" * 10, "Running '%s' with %s" % (test_name, ", ".join(friendly_opts)) + func(*args) + finally: + for (sect_name, opt_name), val in old_config.items(): + manager.options.set(sect_name, opt_name, val) + + ############################################################################### + # "Non-filter" tests are those that don't require us to create messages and + # see them get filtered. def run_nonfilter_tests(manager): # And now some other 'sanity' checks. *************** *** 425,429 **** # reported. num_looked += 1 ! if num_looked % 500 == 0: print " (scanned", num_looked, "messages...)" if not message.IsFilterCandidate() and \ message.msgclass.lower().startswith("ipm.note"): --- 458,462 ---- # reported. num_looked += 1 ! if num_looked % 500 == 0: print " scanned", num_looked, "messages..." if not message.IsFilterCandidate() and \ message.msgclass.lower().startswith("ipm.note"): *************** *** 449,466 **** --- 482,601 ---- msgstore.test_suite_running = True + ############################################################################### + # "Failure" tests - execute some tests while provoking the msgstore to simulate + # various MAPI errors. Although not complete, it does help exercise our code + # paths through the code. + def _restore_mapi_failure(): + import msgstore + msgstore.test_suite_failure = None + msgstore.test_suite_failure_request = None + + def _setup_for_mapi_failure(checkpoint, hr): + import msgstore + assert msgstore.test_suite_running, "msgstore should already know its running" + assert not msgstore.test_suite_failure, "should already have torn down previous failure" + msgstore.test_suite_failure = pythoncom.com_error, \ + (hr, "testsuite generated error", None, -1) + msgstore.test_suite_failure_request = checkpoint + + def _setup_mapi_notfound_failure(checkpoint): + _setup_for_mapi_failure(checkpoint, mapi.MAPI_E_NOT_FOUND) + + def _do_single_failure_ham_test(driver, checkpoint, hr): + _do_single_failure_test(driver, True, checkpoint, hr) + + def _do_single_failure_spam_test(driver, checkpoint, hr): + _do_single_failure_test(driver, False, checkpoint, hr) + + def _do_single_failure_test(driver, is_ham, checkpoint, hr): + print "-> Testing MAPI error '%s' in %s" % (mapiutil.GetScodeString(hr), + checkpoint) + # message moved after we have ID, but before opening. + folder = driver.folder_watch + if is_ham: + msg, words = driver.CreateTestMessageInFolder(HAM, folder) + else: + msg, words = driver.CreateTestMessageInFolder(SPAM, folder) + try: + _setup_for_mapi_failure(checkpoint, hr) + try: + # sleep to ensure filtering. + WaitForFilters() + finally: + _restore_mapi_failure() + if driver.FindTestMessage(folder) is None: + TestFailed("We appear to have filtered a message even though we forced 'not found' failure") + finally: + print "<- Finished MAPI error '%s' in %s" % (mapiutil.GetScodeString(hr), + checkpoint) + if msg is not None: + msg.Delete() + + def do_failure_tests(manager): + # We setup msgstore to fail for us, then try a few tests. The idea is to + # ensure we gracefully degrade in these failures. + # We set verbosity to min of 1, as this helps us see how the filters handle + # the errors. + driver = Driver(manager) + driver.CleanAllTestMessages() + old_verbose = manager.verbose + manager.verbose = max(1, old_verbose) + try: + _do_single_failure_ham_test(driver, "MAPIMsgStoreMsg._EnsureObject", mapi.MAPI_E_NOT_FOUND) + _do_single_failure_ham_test(driver, "MAPIMsgStoreMsg.SetField", -2146644781) + _do_single_failure_ham_test(driver, "MAPIMsgStoreMsg.Save", -2146644781) + # SetReadState??? + _do_single_failure_spam_test(driver, "MAPIMsgStoreMsg._DoCopyMove", mapi.MAPI_E_TABLE_TOO_BIG) + finally: + manager.verbose = old_verbose + + def run_failure_tests(manager): + "Forced MAPI failure tests" + apply_with_new_config(manager, + {"Filter.timer_enabled": True, + }, + do_failure_tests, manager) + apply_with_new_config(manager, + {"Filter.timer_enabled": False, + }, + do_failure_tests, manager) + + def filter_message_with_event(msg, mgr, all_actions=True): + import filter + try: + return filter._original_filter_message(msg, mgr, all_actions) + finally: + filter_event.set() + def test(manager): import msgstore from dialogs import SetWaitCursor SetWaitCursor(1) + + import filter + if "_original_filter_message" not in filter.__dict__: + filter._original_filter_message = filter.filter_message + filter.filter_message = filter_message_with_event + try: # restore the plugin config at exit. + assert not msgstore.test_suite_running, "already running??" msgstore.test_suite_running = True + assert not manager.test_suite_running, "already running??" + manager.test_suite_running = True run_nonfilter_tests(manager) # filtering tests take alot of time - do them last. run_filter_tests(manager) + run_failure_tests(manager) + print "*" * 20 + print "Test suite finished without error!" + print "*" * 20 finally: + print "Restoring standard configuration..." # Always restore configuration to how we started. msgstore.test_suite_running = False + manager.test_suite_running = False manager.LoadConfig() manager.addin.FiltersChanged() # restore original filters. + manager.addin.ProcessMissedMessages() SetWaitCursor(0) From richiehindle at users.sourceforge.net Thu Sep 4 15:30:48 2003 From: richiehindle at users.sourceforge.net (Richie Hindle) Date: Thu Sep 4 17:30:52 2003 Subject: [Spambayes-checkins] spambayes/spambayes __init__.py,1.5,1.6 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv8404 Modified Files: __init__.py Log Message: Uprev to 1.0a5. Index: __init__.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/__init__.py,v retrieving revision 1.5 retrieving revision 1.6 diff -C2 -d -r1.5 -r1.6 *** __init__.py 7 Jul 2003 00:19:31 -0000 1.5 --- __init__.py 4 Sep 2003 21:30:46 -0000 1.6 *************** *** 1,3 **** # package marker. ! __version__ = '1.0a4' --- 1,3 ---- # package marker. ! __version__ = '1.0a5' From mhammond at users.sourceforge.net Thu Sep 4 05:59:07 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Thu Sep 4 18:18:46 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs opt_processors.py, 1.12, 1.13 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs In directory sc8-pr-cvs1:/tmp/cvs-serv18964 Modified Files: opt_processors.py Log Message: Remove stray 'print' Index: opt_processors.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/opt_processors.py,v retrieving revision 1.12 retrieving revision 1.13 diff -C2 -d -r1.12 -r1.13 *** opt_processors.py 26 Aug 2003 06:25:33 -0000 1.12 --- opt_processors.py 4 Sep 2003 11:59:05 -0000 1.13 *************** *** 190,194 **** slider_pos = float(slider_pos) * self.max_val / self.ticks str_val = str(slider_pos) - print "Slider wants to set to", str_val edit = self.GetControl() win32gui.SendMessage(edit, win32con.WM_SETTEXT, 0, str_val) --- 190,193 ---- From anadelonbrin at users.sourceforge.net Thu Sep 4 17:29:49 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Thu Sep 4 19:29:54 2003 Subject: [Spambayes-checkins] website quotes.ht,1.4,1.5 Message-ID: Update of /cvsroot/spambayes/website In directory sc8-pr-cvs1:/tmp/cvs-serv29971 Modified Files: quotes.ht Log Message: More nice things about us, this time from the TechTV article. (Including a positive comment about the documentation!) Index: quotes.ht =================================================================== RCS file: /cvsroot/spambayes/website/quotes.ht,v retrieving revision 1.4 retrieving revision 1.5 diff -C2 -d -r1.4 -r1.5 *** quotes.ht 16 Aug 2003 13:39:17 -0000 1.4 --- quotes.ht 4 Sep 2003 23:29:47 -0000 1.5 *************** *** 51,54 **** --- 51,61 ----

+

+ I have to say I'm very happy with its performance...The documentation + on the SpamBayes site is very good...It is very much worth the time to + train and use SpamBayes. + Chris DiBona in a TechTV article. +

+

What we are saying about us

From mhammond at users.sourceforge.net Thu Sep 4 05:59:37 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Thu Sep 4 19:46:55 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000/installer spambayes_addin.iss, 1.8, 1.9 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000/installer In directory sc8-pr-cvs1:/tmp/cvs-serv19031 Modified Files: spambayes_addin.iss Log Message: 'welcome.html' is now the 'readme' for the installer. Index: spambayes_addin.iss =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/installer/spambayes_addin.iss,v retrieving revision 1.8 retrieving revision 1.9 diff -C2 -d -r1.8 -r1.9 *** spambayes_addin.iss 9 Aug 2003 03:26:16 -0000 1.8 --- spambayes_addin.iss 4 Sep 2003 11:59:35 -0000 1.9 *************** *** 18,22 **** Source: "dist\spambayes_addin.dll"; DestDir: "{app}"; Flags: ignoreversion regserver Source: "dist\*.*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs ! Source: "dist\about.html"; DestDir: "{app}"; Flags: isreadme [UninstallDelete] --- 18,22 ---- Source: "dist\spambayes_addin.dll"; DestDir: "{app}"; Flags: ignoreversion regserver Source: "dist\*.*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs ! Source: "dist\docs\welcome.html"; DestDir: "{app}"; Flags: isreadme [UninstallDelete] From ta-meyer at ihug.co.nz Fri Sep 5 13:05:31 2003 From: ta-meyer at ihug.co.nz (Tony Meyer) Date: Thu Sep 4 20:05:42 2003 Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py, NONE, 1.1 In-Reply-To: <1ED4ECF91CDED24C8D012BCF2B034F13031B74DA@its-xchg4.massey.ac.nz> Message-ID: <1ED4ECF91CDED24C8D012BCF2B034F130212AE19@its-xchg4.massey.ac.nz> > Anyone know if something is up with the SourceForge CVS servers? I've seen other reports of this as well. I don't know for sure, but I imagine that it has something to do with sourceforge getting everything back to the normal status. Their last email update said it was only days away, so they're probably installing something and letting anon cvs hang for periods without updating. =Tony Meyer From anadelonbrin at users.sourceforge.net Thu Sep 4 18:24:50 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Thu Sep 4 20:24:54 2003 Subject: [Spambayes-checkins] spambayes dbExpImp.py, 1.7, NONE hammie.py, 1.46, NONE hammiecli.py, 1.4, NONE hammiefilter.py, 1.19, NONE hammiesrv.py, 1.11, NONE imapfilter.py, 1.54, NONE mailsort.py, 1.7, NONE mboxtrain.py, 1.12, NONE notesfilter.py, 1.3, NONE pop3proxy.py, 1.97, NONE proxytee.py, 1.4, NONE smtpproxy.py, 1.15, NONE unheader.py, 1.10, NONE Message-ID: Update of /cvsroot/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv6818 Removed Files: dbExpImp.py hammie.py hammiecli.py hammiefilter.py hammiesrv.py imapfilter.py mailsort.py mboxtrain.py notesfilter.py pop3proxy.py proxytee.py smtpproxy.py unheader.py Log Message: Step one of the 1.0a6 shuffle. Remove all of the top-level scripts and replace them with sb-* versions in the scripts directory. These are the names that seemed to be the agreement on spambayes-dev - change them quick if you don't like them ;) Note for anyone reading this in the cvs history - for older history than this, you will need to go to the script directory's parent's attic and look at the old files. The names are the same once the sb- prefix is removed, with the exception of: sb-client was hammiecli sb-filter was hammiefilter sb-pop3dnd was sb-overkill sb-server was pop3proxy sb-upload was proxytee sb-xmlrpcserver was hammiesrv --- dbExpImp.py DELETED --- --- hammie.py DELETED --- --- hammiecli.py DELETED --- --- hammiefilter.py DELETED --- --- hammiesrv.py DELETED --- --- imapfilter.py DELETED --- --- mailsort.py DELETED --- --- mboxtrain.py DELETED --- --- notesfilter.py DELETED --- --- pop3proxy.py DELETED --- --- proxytee.py DELETED --- --- smtpproxy.py DELETED --- --- unheader.py DELETED --- From richiehindle at users.sourceforge.net Thu Sep 4 15:35:31 2003 From: richiehindle at users.sourceforge.net (Richie Hindle) Date: Thu Sep 4 20:35:33 2003 Subject: [Spambayes-checkins] website index.ht,1.23,1.24 Message-ID: Update of /cvsroot/spambayes/website In directory sc8-pr-cvs1:/tmp/cvs-serv9311 Modified Files: index.ht Log Message: *Fifth* pre-release available. Index: index.ht =================================================================== RCS file: /cvsroot/spambayes/website/index.ht,v retrieving revision 1.23 retrieving revision 1.24 diff -C2 -d -r1.23 -r1.24 *** index.ht 26 Aug 2003 08:08:57 -0000 1.23 --- index.ht 4 Sep 2003 21:35:29 -0000 1.24 *************** *** 6,10 ****

News

Outlook plugin release 007 is available. See the Windows page for more. Note that the Outlook plugin is now available from Sourceforge, on this project's Files page.

!

Fourth pre-release of spambayes available. See the download page for more.

You may also like to see what other people have been saying about us in the press and elsewhere.

--- 6,10 ----

News

Outlook plugin release 007 is available. See the Windows page for more. Note that the Outlook plugin is now available from Sourceforge, on this project's Files page.

!

Fifth pre-release of spambayes available. See the download page for more.

You may also like to see what other people have been saying about us in the press and elsewhere.

From anadelonbrin at users.sourceforge.net Thu Sep 4 18:24:50 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Thu Sep 4 20:42:29 2003 Subject: [Spambayes-checkins] spambayes/scripts sb-client.py, NONE, 1.1 sb-dbexpimp.py, NONE, 1.1 sb-filter.py, NONE, 1.1 sb-imapfilter.py, NONE, 1.1 sb-mailsort.py, NONE, 1.1 sb-mboxtrain.py, NONE, 1.1 sb-notesfilter.py, NONE, 1.1 sb-pop3dnd.py, NONE, 1.1 sb-server.py, NONE, 1.1 sb-smtpproxy.py, NONE, 1.1 sb-unheader.py, NONE, 1.1 sb-upload.py, NONE, 1.1 sb-xmlrpcserver.py, NONE, 1.1 sb-overkill.py, 1.1, NONE Message-ID: Update of /cvsroot/spambayes/spambayes/scripts In directory sc8-pr-cvs1:/tmp/cvs-serv6818/scripts Added Files: sb-client.py sb-dbexpimp.py sb-filter.py sb-imapfilter.py sb-mailsort.py sb-mboxtrain.py sb-notesfilter.py sb-pop3dnd.py sb-server.py sb-smtpproxy.py sb-unheader.py sb-upload.py sb-xmlrpcserver.py Removed Files: sb-overkill.py Log Message: Step one of the 1.0a6 shuffle. Remove all of the top-level scripts and replace them with sb-* versions in the scripts directory. These are the names that seemed to be the agreement on spambayes-dev - change them quick if you don't like them ;) Note for anyone reading this in the cvs history - for older history than this, you will need to go to the script directory's parent's attic and look at the old files. The names are the same once the sb- prefix is removed, with the exception of: sb-client was hammiecli sb-filter was hammiefilter sb-pop3dnd was sb-overkill sb-server was pop3proxy sb-upload was proxytee sb-xmlrpcserver was hammiesrv --- NEW FILE: sb-client.py --- #! /usr/bin/env python """A client for hammiesrv. Just feed it your mail on stdin, and it spits out the same message with the spambayes score in a new X-Spambayes-Disposition header. """ import xmlrpclib import sys RPCBASE="http://localhost:65000" def main(): msg = sys.stdin.read() try: x = xmlrpclib.ServerProxy(RPCBASE) m = xmlrpclib.Binary(msg) out = x.filter(m) print out.data except: if __debug__: import traceback traceback.print_exc() print msg if __name__ == "__main__": main() --- NEW FILE: sb-dbexpimp.py --- #! /usr/bin/env python """dbExpImp.py - Bayes database export/import Classes: Abstract: This utility has the primary function of exporting and importing a spambayes database into/from a flat file. This is useful in a number of scenarios. Platform portability of database - flat files can be exported and imported across platforms (winduhs and linux, for example) Database implementation changes - databases can survive database implementation upgrades or new database implementations. For example, if a dbm implementation changes between python x.y and python x.y+1... Database reorganization - an export followed by an import reorgs an existing database, improving performance, at least in some database implementations Database sharing - it is possible to distribute particular databases for research purposes, database sharing purposes, or for new users to have a 'seed' database to start with. Database merging - multiple databases can be merged into one quite easily by simply not specifying -n on an import. This will add the two database nham and nspams together (assuming the two databases do not share corpora) and for wordinfo conflicts, will add spamcount and hamcount together. Spambayes software release migration - an export can be executed before a release upgrade, as part of the installation script. Then, after the new software is installed, an import can be executed, which will effectively preserve existing training. This eliminates the need for retraining every time a release is installed. Others? I'm sure I haven't thought of everything... Usage: dbExpImp [options] options: -e : export -i : import -v : verbose mode (some additional diagnostic messages) -f: FN : flat file to export to or import from -d: FN : name of pickled database file to use -D: FN : name of dbm database file to use -m : merge import into an existing database file. This is meaningful only for import. If omitted, a new database file will be created. If specified, the imported wordinfo will be merged into an existing database. Run dbExpImp -h for more information. -h : help Examples: Export pickled mybayes.db into mybayes.db.export as a csv flat file dbExpImp -e -d mybayes.db -f mybayes.db.export Import mybayes.eb.export into a new DBM mybayes.db dbExpImp -i -D mybayes.db -f mybayes.db.export Export, then import (reorganize) new pickled mybayes.db dbExpImp -e -i -n -d mybayes.db -f mybayes.db.export Convert a bayes database from pickle to DBM dbExpImp -e -d abayes.db -f abayes.export dbExpImp -i -D abayes.db -f abayes.export Create a new database (newbayes.db) from two databases (abayes.db, bbayes.db) dbExpImp -e -d abayes.db -f abayes.export dbExpImp -e -d bbayes.db -f bbayes.export dbExpImp -i -d newbayes.db -f abayes.export dbExpImp -i -m -d newbayes.db -f bbayes.export To Do: o Suggestions? """ # This module is part of the spambayes project, which is Copyright 2002 # The Python Software Foundation and is covered by the Python Software # Foundation license. __author__ = "Tim Stone " from __future__ import generators try: True, False except NameError: # Maintain compatibility with Python 2.2 True, False = 1, 0 import spambayes.storage from spambayes.Options import options import sys, os, getopt, errno, re import urllib def runExport(dbFN, useDBM, outFN): if useDBM: bayes = spambayes.storage.DBDictClassifier(dbFN) words = bayes.db.keys() words.remove(bayes.statekey) else: bayes = spambayes.storage.PickledClassifier(dbFN) words = bayes.wordinfo.keys() try: fp = open(outFN, 'w') except IOError, e: if e.errno != errno.ENOENT: raise nham = bayes.nham; nspam = bayes.nspam; print "Exporting database %s to file %s" % (dbFN, outFN) print "Database has %s ham, %s spam, and %s words" \ % (nham, nspam, len(words)) fp.write("%s,%s,\n" % (nham, nspam)) for word in words: wi = bayes._wordinfoget(word) hamcount = wi.hamcount spamcount = wi.spamcount word = urllib.quote(word) fp.write("%s`%s`%s`\n" % (word, hamcount, spamcount)) fp.close() def runImport(dbFN, useDBM, newDBM, inFN): if newDBM: try: os.unlink(dbFN) except OSError, e: if e.errno != 2: # errno. raise try: os.unlink(dbFN+".dat") except OSError, e: if e.errno != 2: # errno. raise try: os.unlink(dbFN+".dir") except OSError, e: if e.errno != 2: # errno. raise if useDBM: bayes = spambayes.storage.DBDictClassifier(dbFN) else: bayes = spambayes.storage.PickledClassifier(dbFN) try: fp = open(inFN, 'r') except IOError, e: if e.errno != errno.ENOENT: raise nline = fp.readline() (nham, nspam, junk) = re.split(',', nline) if newDBM: bayes.nham = int(nham) bayes.nspam = int(nspam) else: bayes.nham += int(nham) bayes.nspam += int(nspam) if newDBM: impType = "Importing" else: impType = "Merging" print "%s database %s using file %s" % (impType, dbFN, inFN) lines = fp.readlines() for line in lines: (word, hamcount, spamcount, junk) = re.split('`', line) word = urllib.unquote(word) try: wi = bayes.wordinfo[word] except KeyError: wi = bayes.WordInfoClass() wi.hamcount += int(hamcount) wi.spamcount += int(spamcount) bayes._wordinfoset(word, wi) fp.close() print "Storing database, please be patient. Even moderately large" print "databases may take a very long time to store." bayes.store() print "Finished storing database" if useDBM: words = bayes.db.keys() words.remove(bayes.statekey) else: words = bayes.wordinfo.keys() print "Database has %s ham, %s spam, and %s words" \ % (bayes.nham, bayes.nspam, len(words)) if __name__ == '__main__': try: opts, args = getopt.getopt(sys.argv[1:], 'iehmvd:D:f:') except getopt.error, msg: print >>sys.stderr, str(msg) + '\n\n' + __doc__ sys.exit() usePickle = False useDBM = False newDBM = True dbFN = None flatFN = None exp = False imp = False for opt, arg in opts: if opt == '-h': print >>sys.stderr, __doc__ sys.exit() elif opt == '-d': useDBM = False dbFN = arg elif opt == '-D': useDBM = True dbFN = arg elif opt == '-f': flatFN = arg elif opt == '-e': exp = True elif opt == '-i': imp = True elif opt == '-m': newDBM = False elif opt == '-v': options["globals", "verbose"] = True if (dbFN and flatFN): if exp: runExport(dbFN, useDBM, flatFN) if imp: runImport(dbFN, useDBM, newDBM, flatFN) else: print >>sys.stderr, __doc__ --- NEW FILE: sb-filter.py --- #!/usr/bin/env python ## A hammie front-end to make the simple stuff simple. ## ## ## The intent is to call this from procmail and its ilk like so: ## ## :0 fw ## | hammiefilter.py ## ## Then, you can set up your MUA to pipe ham and spam to it, one at a ## time, by calling it with either the -g or -s options, respectively. ## ## Author: Neale Pickett ## """Usage: %(program)s [OPTION]... [OPTION] is one of: -h show usage and exit -x show some usage examples and exit -d DBFILE use database in DBFILE -D PICKLEFILE use pickle (instead of database) in PICKLEFILE -n create a new database *+ -f filter (default if no processing options are given) *+ -t [EXPERIMENTAL] filter and train based on the result (you must make sure to untrain all mistakes later) * -g [EXPERIMENTAL] (re)train as a good (ham) message * -s [EXPERIMENTAL] (re)train as a bad (spam) message * -G [EXPERIMENTAL] untrain ham (only use if you've already trained this message) * -S [EXPERIMENTAL] untrain spam (only use if you've already trained this message) All options marked with '*' operate on stdin. Only those processing options marked with '+' send a modified message to stdout. """ import os import sys import getopt from spambayes import hammie, Options, mboxutils # See Options.py for explanations of these properties program = sys.argv[0] example_doc = """_Examples_ filter a message on disk: %(program)s < message (re)train a message as ham: %(program)s -g < message (re)train a message as spam: %(program)s -s < message procmail recipe to filter and train in one step: :0 fw | %(program)s -t mutt configuration. This binds the 'H' key to retrain the message as ham, and prompt for a folder to move it to. The 'S' key retrains as spam, and moves to a 'spam' folder. XXX: add this """ def examples(): print example_doc % globals() sys.exit(0) def usage(code, msg=''): """Print usage message and sys.exit(code).""" if msg: print >> sys.stderr, msg print >> sys.stderr print >> sys.stderr, __doc__ % globals() sys.exit(code) class HammieFilter(object): def __init__(self): options = Options.options # This is a bit of a hack to counter the default for # persistent_storage_file changing from ~/.hammiedb to hammie.db # This will work unless a user: # * had hammie.db as their value for persistent_storage_file, and # * their config file was loaded by Options.py. if options["Storage", "persistent_storage_file"] == \ options.default("Storage", "persistent_storage_file"): options["Storage", "persistent_storage_file"] = \ "~/.hammiedb" options.merge_files(['/etc/hammierc', os.path.expanduser('~/.hammierc')]) self.dbname = options["Storage", "persistent_storage_file"] self.dbname = os.path.expanduser(self.dbname) self.usedb = options["Storage", "persistent_use_database"] def newdb(self): h = hammie.open(self.dbname, self.usedb, 'n') h.store() print >> sys.stderr, "Created new database in", self.dbname def filter(self, msg): h = hammie.open(self.dbname, self.usedb, 'r') return h.filter(msg) def filter_train(self, msg): h = hammie.open(self.dbname, self.usedb, 'c') return h.filter(msg, train=True) def train_ham(self, msg): h = hammie.open(self.dbname, self.usedb, 'c') h.train_ham(msg, True) h.store() def train_spam(self, msg): h = hammie.open(self.dbname, self.usedb, 'c') h.train_spam(msg, True) h.store() def untrain_ham(self, msg): h = hammie.open(self.dbname, self.usedb, 'c') h.untrain_ham(msg) h.store() def untrain_spam(self, msg): h = hammie.open(self.dbname, self.usedb, 'c') h.untrain_spam(msg) h.store() def main(): h = HammieFilter() actions = [] opts, args = getopt.getopt(sys.argv[1:], 'hxd:D:nfgstGS', ['help', 'examples']) for opt, arg in opts: if opt in ('-h', '--help'): usage(0) elif opt in ('-x', '--examples'): examples() elif opt == '-d': h.usedb = True h.dbname = arg elif opt == '-D': h.usedb = False h.dbname = arg elif opt == '-f': actions.append(h.filter) elif opt == '-g': actions.append(h.train_ham) elif opt == '-s': actions.append(h.train_spam) elif opt == '-t': actions.append(h.filter_train) elif opt == '-G': actions.append(h.untrain_ham) elif opt == '-S': actions.append(h.untrain_spam) elif opt == "-n": h.newdb() sys.exit(0) if actions == []: actions = [h.filter] msg = mboxutils.get_message(sys.stdin) for action in actions: action(msg) sys.stdout.write(msg.as_string(unixfrom=(msg.get_unixfrom() is not None))) if __name__ == "__main__": main() --- NEW FILE: sb-imapfilter.py --- #!/usr/bin/env python """An IMAP filter. An IMAP message box is scanned and all non-scored messages are scored and (where necessary) filtered. The original filter design owed much to isbg by Roger Binns (http://www.rogerbinns.com/isbg). Usage: imapfilter [options] note: option values with spaces in them must be enclosed in double quotes options: -d dbname : pickled training database filename -D dbname : dbm training database filename -t : train contents of spam folder and ham folder -c : classify inbox -h : help -v : verbose mode -p : security option to prompt for imap password, rather than look in options["imap", "password"] -e y/n : expunge/purge messages on exit (y) or not (n) -i debuglvl : a somewhat mysterious imaplib debugging level -l minutes : period of time between filtering operations -b : Launch a web browser showing the user interface. Examples: Classify inbox, with dbm database imapfilter -c -D bayes.db Train Spam and Ham, then classify inbox, with dbm database imapfilter -t -c -D bayes.db Train Spam and Ham only, with pickled database imapfilter -t -d bayes.db Warnings: o This is alpha software! The filter is currently being developed and tested. We do *not* recommend using it on a production system unless you are confident that you can get your mail back if you lose it. On the other hand, we do recommend that you test it for us and let us know if anything does go wrong. o By default, the filter does *not* delete, modify or move any of your mail. Due to quirks in how imap works, new versions of your mail are modified and placed in new folders, but the originals are still available. These are flagged with the /Deleted flag so that you know that they can be removed. Your mailer may not show these messages by default, but there should be an option to do so. *However*, if your mailer automatically purges/expunges (i.e. permanently deletes) mail flagged as such, *or* if you set the imap_expunge option to True, then this mail will be irretrievably lost. To Do: o IMAPMessage and IMAPFolder currently carry out very simple checks of responses received from IMAP commands, but if the response is not "OK", then the filter terminates. Handling of these errors could be much nicer. o IMAP over SSL is untested. o Develop a test script, like spambayes/test/test_pop3proxy.py that runs through some tests (perhaps with a *real* imap server, rather than a dummy one). This would make it easier to carry out the tests against each server whenever a change is made. o IMAP supports authentication via other methods than the plain-text password method that we are using at the moment. Neither of the servers I have access to offer any alternative method, however. If someone's does, then it would be nice to offer this. o Usernames should be able to be literals as well as quoted strings. This might help if the username/password has special characters like accented characters. o Suggestions? """ # This module is part of the spambayes project, which is Copyright 2002-3 # The Python Software Foundation and is covered by the Python Software # Foundation license. __author__ = "Tony Meyer , Tim Stone" __credits__ = "All the Spambayes folk." from __future__ import generators try: True, False except NameError: # Maintain compatibility with Python 2.2 True, False = 1, 0 import socket import os import re import time import sys import getopt import types import email import email.Parser from getpass import getpass from email.Utils import parsedate from spambayes.Options import options from spambayes import tokenizer, storage, message, Dibbler from spambayes.UserInterface import UserInterfaceServer from spambayes.ImapUI import IMAPUserInterface from spambayes.Version import get_version_string from imaplib import IMAP4 from imaplib import Time2Internaldate try: if options["imap", "use_ssl"]: from imaplib import IMAP_SSL as BaseIMAP else: from imaplib import IMAP4 as BaseIMAP except ImportError: from imaplib import IMAP4 as BaseIMAP # global IMAPlib object global imap imap = None # A flag can have any character in the ascii range 32-126 # except for (){ %*"\ FLAG_CHARS = "" for i in range(32, 127): if not chr(i) in ['(', ')', '{', ' ', '%', '*', '"', '\\']: FLAG_CHARS += chr(i) FLAG = r"\\?[" + re.escape(FLAG_CHARS) + r"]+" # The empty flag set "()" doesn't match, so that extract returns # data["FLAGS"] == None FLAGS_RE = re.compile(r"(FLAGS) (\((" + FLAG + r" )*(" + FLAG + r")\))") INTERNALDATE_RE = re.compile(r"(INTERNALDATE) (\"\d{1,2}\-[A-Za-z]{3,3}\-" + r"\d{2,4} \d{2,2}\:\d{2,2}\:\d{2,2} " + r"[\+\-]\d{4,4}\")") RFC822_RE = re.compile(r"(RFC822) (\{[\d]+\})") RFC822_HEADER_RE = re.compile(r"(RFC822.HEADER) (\{[\d]+\})") UID_RE = re.compile(r"(UID) ([\d]+)") FETCH_RESPONSE_RE = re.compile(r"([0-9]+) \(([" + \ re.escape(FLAG_CHARS) + r"\"\{\}\(\)\\ ]*)\)?") LITERAL_RE = re.compile(r"^\{[\d]+\}$") def _extract_fetch_data(response): '''Extract data from the response given to an IMAP FETCH command.''' # Response might be a tuple containing literal data # At the moment, we only handle one literal per response. This # may need to be improved if our code ever asks for something # more complicated (like RFC822.Header and RFC822.Body) if type(response) == types.TupleType: literal = response[1] response = response[0] else: literal = None # the first item will always be the message number mo = FETCH_RESPONSE_RE.match(response) data = {} if mo is None: print """IMAP server gave strange fetch response. Please report this as a bug.""" print response else: data["message_number"] = mo.group(1) response = mo.group(2) # We support the following FETCH items: # FLAGS # INTERNALDATE # RFC822 # UID # RFC822.HEADER # All others are ignored. for r in [FLAGS_RE, INTERNALDATE_RE, RFC822_RE, UID_RE, RFC822_HEADER_RE]: mo = r.search(response) if mo is not None: if LITERAL_RE.match(mo.group(2)): data[mo.group(1)] = literal else: data[mo.group(1)] = mo.group(2) return data class IMAPSession(BaseIMAP): '''A class extending the IMAP4 class, with a few optimizations''' def __init__(self, server, port, debug=0, do_expunge=False): try: BaseIMAP.__init__(self, server, port) except: # A more specific except would be good here, but I get # (in Python 2.2) a generic 'error' and a 'gaierror' # if I pass a valid domain that isn't an IMAP server # or invalid domain (respectively) print "Invalid server or port, please check these settings." sys.exit(-1) self.debug = debug # For efficiency, we remember which folder we are currently # in, and only send a select command to the IMAP server if # we want to *change* folders. This function is used by # both IMAPMessage and IMAPFolder. self.current_folder = None self.do_expunge = do_expunge def login(self, username, pwd): try: BaseIMAP.login(self, username, pwd) # superclass login except BaseIMAP.error, e: if str(e) == "permission denied": print "There was an error logging in to the IMAP server." print "The userid and/or password may be incorrect." sys.exit() else: raise def logout(self): # sign off if self.do_expunge: self.expunge() BaseIMAP.logout(self) # superclass logout def SelectFolder(self, folder): '''A method to point ensuing imap operations at a target folder''' if self.current_folder != folder: if self.current_folder != None: if self.do_expunge: # It is faster to do close() than a single # expunge when we log out (because expunge returns # a list of all the deleted messages, that we don't do # anything with) imap.close() # We *always* use SELECT and not EXAMINE, because this # speeds things up considerably. response = self.select(folder, None) if response[0] != "OK": print "Invalid response to select %s:\n%s" % (folder, response) sys.exit(-1) self.current_folder = folder return response def folder_list(self): '''Return a alphabetical list of all folders available on the server''' response = self.list() if response[0] != "OK": return [] all_folders = response[1] folders = [] for fol in all_folders: # Sigh. Some servers may give us back the folder name as a # literal, so we need to crunch this out. if isinstance(fol, ()): r = re.compile(r"{\d+}") m = r.search(fol[0]) if not m: # Something is wrong here! Skip this folder continue fol = '%s"%s"' % (fol[0][:m.start()], fol[1]) r = re.compile(r"\(([\w\\ ]*)\) ") m = r.search(fol) if not m: # Something is not good with this folder, so skip it. continue name_attributes = fol[:m.end()-1] # IMAP is a truly odd protocol. The delimiter is # only the delimiter for this particular folder - each # folder *may* have a different delimiter self.folder_delimiter = fol[m.end()+1:m.end()+2] # a bit of a hack, but we really need to know if this is # the case if self.folder_delimiter == ',': print """WARNING: Your imap server uses commas as the folder delimiter. This may cause unpredictable errors.""" folders.append(fol[m.end()+5:-1]) folders.sort() return folders def FindMessage(self, id): '''A (potentially very expensive) method to find a message with a given spambayes id (header), and return a message object (no substance).''' # If efficiency becomes a concern, what we could do is store a # dict of key-to-folder, and look in that folder first. (It might # have moved independantly of us, so we would still have to search # if we didn't find it). For the moment, we do an search through # all folders, alphabetically. for folder_name in self.folder_list(): fol = IMAPFolder(folder_name) for msg in fol: if msg.id == id: return msg return None class IMAPMessage(message.SBHeaderMessage): def __init__(self): message.Message.__init__(self) self.folder = None self.previous_folder = None self.rfc822_command = "RFC822.PEEK" self.got_substance = False def setFolder(self, folder): self.folder = folder def _check(self, response, command): if response[0] != "OK": print "Invalid response to %s:\n%s" % (command, response) sys.exit(-1) def extractTime(self): # When we create a new copy of a message, we need to specify # a timestamp for the message. If the message has a valid date # header we use that. Otherwise, we use the current time. message_date = self["Date"] if message_date is not None: parsed_date = parsedate(message_date) if parsed_date is not None: return Time2Internaldate(time.mktime(parsed_date)) else: return Time2Internaldate(time.time()) def get_substance(self): '''Retrieve the RFC822 message from the IMAP server and set as the substance of this message.''' if self.got_substance: return if not self.uid or not self.id: print "Cannot get substance of message without an id and an UID" return imap.SelectFolder(self.folder.name) # We really want to use RFC822.PEEK here, as that doesn't effect # the status of the message. Unfortunately, it appears that not # all IMAP servers support this, even though it is in RFC1730 # Actually, it's not: we should be using BODY.PEEK try: response = imap.uid("FETCH", self.uid, self.rfc822_command) except IMAP4.error: self.rfc822_command = "RFC822" response = imap.uid("FETCH", self.uid, self.rfc822_command) if response[0] != "OK": self.rfc822_command = "RFC822" response = imap.uid("FETCH", self.uid, self.rfc822_command) self._check(response, "uid fetch") data = _extract_fetch_data(response[1][0]) # Annoyingly, we can't just pass over the RFC822 message to an # existing message object (like self) and have it parse it. So # we go through the hoops of creating a new message, and then # copying over all its internals. new_msg = email.Parser.Parser().parsestr(data["RFC822"]) self._headers = new_msg._headers self._unixfrom = new_msg._unixfrom self._payload = new_msg._payload self._charset = new_msg._charset self.preamble = new_msg.preamble self.epilogue = new_msg.epilogue self._default_type = new_msg._default_type if not self.has_key(options["Headers", "mailid_header_name"]): self[options["Headers", "mailid_header_name"]] = self.id self.got_substance = True if options["globals", "verbose"]: sys.stdout.write(chr(8) + "*") def MoveTo(self, dest): '''Note that message should move to another folder. No move is carried out until Save() is called, for efficiency.''' if self.previous_folder is None: self.previous_folder = self.folder self.folder = dest def Save(self): '''Save message to imap server.''' # we can't actually update the message with IMAP # so what we do is create a new message and delete the old one if self.folder is None: raise RuntimeError, """Can't save a message that doesn't have a folder.""" if not self.id: raise RuntimeError, """Can't save a message that doesn't have an id.""" response = imap.uid("FETCH", self.uid, "(FLAGS INTERNALDATE)") self._check(response, 'fetch (flags internaldate)') data = _extract_fetch_data(response[1][0]) if data.has_key("INTERNALDATE"): msg_time = data["INTERNALDATE"] else: msg_time = self.extractTime() if data.has_key("FLAGS"): flags = data["FLAGS"] # The \Recent flag can be fetched, but cannot be stored # We must remove it from the list if it is there. flags = re.sub(r"\\Recent ?|\\ ?Recent", "", flags) else: flags = None response = imap.append(self.folder.name, flags, msg_time, self.as_string()) if response[0] == "NO": # This may be because we have tried to set an invalid flag. # Try again, losing all the flag information, but warn the # user that this has happened. response = imap.append(self.folder.name, None, msg_time, self.as_string()) if response[0] == "OK": print "WARNING: Could not append flags: %s" % (flags,) self._check(response, 'append') if self.previous_folder is None: imap.SelectFolder(self.folder.name) else: imap.SelectFolder(self.previous_folder.name) self.previous_folder = None response = imap.uid("STORE", self.uid, "+FLAGS.SILENT", "(\\Deleted)") self._check(response, 'store') # We need to update the uid, as it will have changed. # Although we don't use the UID to keep track of messages, we do # have to use it for IMAP operations. imap.SelectFolder(self.folder.name) response = imap.uid("SEARCH", "(UNDELETED HEADER " + \ options["Headers", "mailid_header_name"] + \ " " + self.id + ")") self._check(response, 'search') new_id = response[1][0] # Let's hope it doesn't, but, just in case, if the search # turns up empty, we make the assumption that the new # message is the last one with a recent flag if new_id == "": response = imap.uid("SEARCH", "RECENT") new_id = response[1][0] if new_id.find(' ') > -1: ids = new_id.split(' ') new_id = ids[-1] # Ok, now we're in trouble if we still haven't found it. # We make a huge assumption that the new message is the one # with the highest UID (they are sequential, so this will be # ok as long as another message hasn't also arrived) if new_id == "": response = imap.uid("SEARCH", "ALL") new_id = response[1][0] if new_id.find(' ') > -1: ids = new_id.split(' ') new_id = ids[-1] self.uid = new_id # This performs a similar function to email.message_from_string() def imapmessage_from_string(s, _class=IMAPMessage, strict=False): return email.message_from_string(s, _class, strict) class IMAPFolder(object): def __init__(self, folder_name): self.name = folder_name # Unique names for cached messages - see _generate_id below. self.lastBaseMessageName = '' self.uniquifier = 2 def __cmp__(self, obj): '''Two folders are equal if their names are equal''' if obj is None: return False return cmp(self.name, obj.name) def _check(self, response, command): if response[0] != "OK": print "Invalid response to %s:\n%s" % (command, response) sys.exit(-1) def __iter__(self): '''IMAPFolder is iterable''' for key in self.keys(): try: yield self[key] except KeyError: pass def recent_uids(self): '''Returns uids for all the messages in the folder that are flagged as recent, but not flagged as deleted.''' imap.SelectFolder(self.name, True) response = imap.uid("SEARCH", "RECENT UNDELETED") self._check(response, "SEARCH RECENT UNDELETED") return response[1][0].split(' ') def keys(self): '''Returns *uids* for all the messages in the folder not marked as deleted.''' imap.SelectFolder(self.name) response = imap.uid("SEARCH", "UNDELETED") self._check(response, "SEARCH UNDELETED") if response[1][0] == "": return [] return response[1][0].split(' ') def __getitem__(self, key): '''Return message (no substance) matching the given *uid*.''' # We don't retrieve the substances of the message here - you need # to call msg.get_substance() to do that. imap.SelectFolder(self.name) # Using RFC822.HEADER.LINES would be better here, but it seems # that not all servers accept it, even though it is in the RFC response = imap.uid("FETCH", key, "RFC822.HEADER") self._check(response, "uid fetch header") data = _extract_fetch_data(response[1][0]) msg = IMAPMessage() msg.setFolder(self) msg.uid = key r = re.compile(re.escape(options["Headers", "mailid_header_name"]) + \ "\:\s*(\d+(\-\d)?)") mo = r.search(data["RFC822.HEADER"]) if mo is None: msg.setId(self._generate_id()) # Unfortunately, we now have to re-save this message, so that # our id is stored on the IMAP server. Before anyone suggests # it, we can't store it as a flag, because user-defined flags # aren't supported by all IMAP servers. # This will need to be done once per message. msg.get_substance() msg.Save() else: msg.setId(mo.group(1)) if options["globals", "verbose"]: sys.stdout.write(".") return msg # Lifted straight from pop3proxy.py (under the name getNewMessageName) def _generate_id(self): # The message id is the time it arrived, with a uniquifier # appended if two arrive within one clock tick of each other. messageName = "%10.10d" % long(time.time()) if messageName == self.lastBaseMessageName: messageName = "%s-%d" % (messageName, self.uniquifier) self.uniquifier += 1 else: self.lastBaseMessageName = messageName self.uniquifier = 2 return messageName def Train(self, classifier, isSpam): '''Train folder as spam/ham''' num_trained = 0 for msg in self: if msg.GetTrained() == (not isSpam): msg.get_substance() msg.delSBHeaders() classifier.unlearn(msg.asTokens(), not isSpam) # Once the message has been untrained, it's training memory # should reflect that on the off chance that for some reason # the training breaks, which happens on occasion (the # tokenizer is not yet perfect) msg.RememberTrained(None) if msg.GetTrained() is None: msg.get_substance() msg.delSBHeaders() classifier.learn(msg.asTokens(), isSpam) num_trained += 1 msg.RememberTrained(isSpam) if isSpam: move_opt_name = "move_trained_spam_to_folder" else: move_opt_name = "move_trained_ham_to_folder" if options["imap", move_opt_name] != "": msg.MoveTo(IMAPFolder(options["imap", move_opt_name])) msg.Save() return num_trained def Filter(self, classifier, spamfolder, unsurefolder): count = {} count["ham"] = 0 count["spam"] = 0 count["unsure"] = 0 for msg in self: if msg.GetClassification() is None: msg.get_substance() (prob, clues) = classifier.spamprob(msg.asTokens(), evidence=True) # add headers and remember classification msg.addSBHeaders(prob, clues) cls = msg.GetClassification() if cls == options["Hammie", "header_ham_string"]: # we leave ham alone count["ham"] += 1 elif cls == options["Hammie", "header_spam_string"]: msg.MoveTo(spamfolder) count["spam"] += 1 else: msg.MoveTo(unsurefolder) count["unsure"] += 1 msg.Save() return count class IMAPFilter(object): def __init__(self, classifier): self.spam_folder = IMAPFolder(options["imap", "spam_folder"]) self.unsure_folder = IMAPFolder(options["imap", "unsure_folder"]) self.classifier = classifier def Train(self): if options["globals", "verbose"]: t = time.time() total_ham_trained = 0 total_spam_trained = 0 if options["imap", "ham_train_folders"] != "": ham_training_folders = options["imap", "ham_train_folders"] for fol in ham_training_folders: # Select the folder to make sure it exists imap.SelectFolder(fol) if options['globals', 'verbose']: print " Training ham folder %s" % (fol) folder = IMAPFolder(fol) num_ham_trained = folder.Train(self.classifier, False) total_ham_trained += num_ham_trained if options['globals', 'verbose']: print " %s trained." % (num_ham_trained) if options["imap", "spam_train_folders"] != "": spam_training_folders = options["imap", "spam_train_folders"] for fol in spam_training_folders: # Select the folder to make sure it exists imap.SelectFolder(fol) if options['globals', 'verbose']: print " Training spam folder %s" % (fol) folder = IMAPFolder(fol) num_spam_trained = folder.Train(self.classifier, True) total_spam_trained += num_spam_trained if options['globals', 'verbose']: print " %s trained." % (num_spam_trained) if total_ham_trained or total_spam_trained: self.classifier.store() if options["globals", "verbose"]: print "Training took %s seconds, %s messages were trained" \ % (time.time() - t, total_ham_trained + total_spam_trained) def Filter(self): if options["globals", "verbose"]: t = time.time() count = None # Select the spam folder and unsure folder to make sure they exist imap.SelectFolder(self.spam_folder.name) imap.SelectFolder(self.unsure_folder.name) for filter_folder in options["imap", "filter_folders"]: # Select the folder to make sure it exists imap.SelectFolder(filter_folder) folder = IMAPFolder(filter_folder) count = folder.Filter(self.classifier, self.spam_folder, self.unsure_folder) if options["globals", "verbose"]: if count is not None: print "\nClassified %s ham, %s spam, and %s unsure." % \ (count["ham"], count["spam"], count["unsure"]) print "Classifying took", time.time() - t, "seconds." def run(): global imap try: opts, args = getopt.getopt(sys.argv[1:], 'hbtcvpl:e:i:d:D:') except getopt.error, msg: print >>sys.stderr, str(msg) + '\n\n' + __doc__ sys.exit() bdbname = options["Storage", "persistent_storage_file"] useDBM = options["Storage", "persistent_use_database"] doTrain = False doClassify = False doExpunge = options["imap", "expunge"] imapDebug = 0 sleepTime = 0 promptForPass = False launchUI = False server = "" username = "" for opt, arg in opts: if opt == '-h': print >>sys.stderr, __doc__ sys.exit() elif opt == '-d': useDBM = False bdbname = arg elif opt == '-D': useDBM = True bdbname = arg elif opt == "-b": launchUI = True elif opt == '-t': doTrain = True elif opt == '-p': promptForPass = True elif opt == '-c': doClassify = True elif opt == '-v': options["globals", "verbose"] = True elif opt == '-e': if arg == 'y': doExpunge = True else: doExpunge = False elif opt == '-i': imapDebug = int(arg) elif opt == '-l': sleepTime = int(arg) * 60 # Let the user know what they are using... print get_version_string("IMAP Filter") print "and engine %s.\n" % (get_version_string(),) if not (doClassify or doTrain or launchUI): print "-b, -c, or -t operands must be specified." print "Please use the -h operand for help." sys.exit() if (launchUI and (doClassify or doTrain)): print """ -b option is exclusive with -c and -t options. The user interface will be launched, but no classification or training will be performed.""" bdbname = os.path.expanduser(bdbname) if options["globals", "verbose"]: print "Loading database %s..." % (bdbname), classifier = storage.open_storage(bdbname, useDBM) if options["globals", "verbose"]: print "Done." if options["imap", "server"]: # The options class is ahead of us here: # it knows that imap:server will eventually be able to have # multiple values, but for the moment, we just use the first one server = options["imap", "server"] if len(server) > 0: server = server[0] username = options["imap", "username"] if len(username) > 0: username = username[0] if not promptForPass: pwd = options["imap", "password"] if len(pwd) > 0: pwd = pwd[0] else: pwd = None if not launchUI: print "You need to specify both a server and a username." sys.exit() if promptForPass: pwd = getpass() if server.find(':') > -1: server, port = server.split(':', 1) port = int(port) else: if options["imap", "use_ssl"]: port = 993 else: port = 143 imap_filter = IMAPFilter(classifier) # Web interface if launchUI: if server != "": imap = IMAPSession(server, port, imapDebug, doExpunge) httpServer = UserInterfaceServer(options["html_ui", "port"]) httpServer.register(IMAPUserInterface(classifier, imap, pwd)) Dibbler.run(launchBrowser=launchUI) else: while True: imap = IMAPSession(server, port, imapDebug, doExpunge) imap.login(username, pwd) if doTrain: if options["globals", "verbose"]: print "Training" imap_filter.Train() if doClassify: if options["globals", "verbose"]: print "Classifying" imap_filter.Filter() imap.logout() if sleepTime: time.sleep(sleepTime) else: break if __name__ == '__main__': run() --- NEW FILE: sb-mailsort.py --- #! /usr/bin/env python """\ To train: %(program)s -t ham.mbox spam.mbox To filter mail (using .forward or .qmail): |%(program)s Maildir/ Mail/Spam/ To print the score and top evidence for a message or messages: %(program)s -s message [message ...] """ SPAM_CUTOFF = 0.57 SIZE_LIMIT = 5000000 # messages larger are not analyzed BLOCK_SIZE = 10000 RC_DIR = "~/.spambayes" DB_FILE = RC_DIR + "/wordprobs.cdb" CONFIG_FILE = RC_DIR + "/bayescustomize.ini" import sys import os import getopt import email import time import signal import socket import email DB_FILE = os.path.expanduser(DB_FILE) def import_spambayes(): global mboxutils, CdbClassifier, tokenize if not os.environ.has_key('BAYESCUSTOMIZE'): os.environ['BAYESCUSTOMIZE'] = os.path.expanduser(CONFIG_FILE) from spambayes import mboxutils from spambayes.cdb_classifier import CdbClassifier from spambayes.tokenizer import tokenize try: True, False except NameError: # Maintain compatibility with Python 2.2 True, False = 1, 0 program = sys.argv[0] # For usage(); referenced by docstring above def usage(code, msg=''): """Print usage message and sys.exit(code).""" if msg: print >> sys.stderr, msg print >> sys.stderr print >> sys.stderr, __doc__ % globals() sys.exit(code) def maketmp(dir): hostname = socket.gethostname() pid = os.getpid() fd = -1 for x in xrange(200): filename = "%d.%d.%s" % (time.time(), pid, hostname) pathname = "%s/tmp/%s" % (dir, filename) try: fd = os.open(pathname, os.O_WRONLY|os.O_CREAT|os.O_EXCL, 0600) except IOError, exc: if exc[i] not in (errno.EINT, errno.EEXIST): raise else: break time.sleep(2) if fd == -1: raise SystemExit, "could not create a mail file" return (os.fdopen(fd, "wb"), pathname, filename) def train(bayes, msgs, is_spam): """Train bayes with all messages from a mailbox.""" mbox = mboxutils.getmbox(msgs) for msg in mbox: bayes.learn(tokenize(msg), is_spam) def train_messages(ham_name, spam_name): """Create database using messages.""" rc_dir = os.path.expanduser(RC_DIR) if not os.path.exists(rc_dir): print "Creating", RC_DIR, "directory..." os.mkdir(rc_dir) bayes = CdbClassifier() print 'Training with ham...' train(bayes, ham_name, False) print 'Training with spam...' train(bayes, spam_name, True) print 'Update probabilities and writing DB...' db = open(DB_FILE, "wb") bayes.save_wordinfo(db) db.close() print 'done' def filter_message(hamdir, spamdir): signal.signal(signal.SIGALRM, lambda s: sys.exit(1)) signal.alarm(24 * 60 * 60) # write message to temporary file (must be on same partition) tmpfile, pathname, filename = maketmp(hamdir) try: tmpfile.write(os.environ.get("DTLINE", "")) # delivered-to line bytes = 0 blocks = [] while 1: block = sys.stdin.read(BLOCK_SIZE) if not block: break bytes += len(block) if bytes < SIZE_LIMIT: blocks.append(block) tmpfile.write(block) tmpfile.close() if bytes < SIZE_LIMIT: msgdata = ''.join(blocks) del blocks msg = email.message_from_string(msgdata) del msgdata bayes = CdbClassifier(open(DB_FILE, 'rb')) prob = bayes.spamprob(tokenize(msg)) else: prob = 0.0 if prob > SPAM_CUTOFF: os.rename(pathname, "%s/new/%s" % (spamdir, filename)) else: os.rename(pathname, "%s/new/%s" % (hamdir, filename)) except: os.unlink(pathname) raise def print_message_score(msg_name, msg_fp): msg = email.message_from_file(msg_fp) bayes = CdbClassifier(open(DB_FILE, 'rb')) prob, evidence = bayes.spamprob(tokenize(msg), evidence=True) print msg_name, prob for word, prob in evidence: print ' ', `word`, prob def main(): global DB_FILE, CONFIG_FILE try: opts, args = getopt.getopt(sys.argv[1:], 'tsd:c:') except getopt.error, msg: usage(2, msg) mode = 'sort' for opt, val in opts: if opt == '-t': mode = 'train' elif opt == '-s': mode = 'score' elif opt == '-d': DB_FILE = val elif opt == '-c': CONFIG_FILE = val else: assert 0, 'invalid option' import_spambayes() if mode == 'sort': if len(args) != 2: usage(2, 'wrong number of arguments') filter_message(args[0], args[1]) elif mode == 'train': if len(args) != 2: usage(2, 'wrong number of arguments') train_messages(args[0], args[1]) elif mode == 'score': if args: for msg in args: print_message_score(msg, open(msg)) else: print_message_score('', sys.stdin) if __name__ == "__main__": main() --- NEW FILE: sb-mboxtrain.py --- #! /usr/bin/env python ### Train spambayes on all previously-untrained messages in a mailbox. ### ### This keeps track of messages it's already trained by adding an ### X-Spambayes-Trained: header to each one. Then, if you move one to ### another folder, it will retrain that message. You would want to run ### this from a cron job on your server. """Usage: %(program)s [OPTIONS] ... Where OPTIONS is one or more of: -h show usage and exit -d DBNAME use the DBM store. A DBM file is larger than the pickle and creating it is slower, but loading it is much faster, especially for large word databases. Recommended for use with hammiefilter or any procmail-based filter. -D DBNAME use the pickle store. A pickle is smaller and faster to create, but much slower to load. Recommended for use with pop3proxy and hammiesrv. -g PATH mbox or directory of known good messages (non-spam) to train on. Can be specified more than once. -s PATH mbox or directory of known spam messages to train on. Can be specified more than once. -f force training, ignoring the trained header. Use this if you need to rebuild your database from scratch. -q quiet mode; no output -n train mail residing in "new" directory, in addition to "cur" directory, which is always trained (Maildir only) -r remove mail which was trained on (Maildir only) """ try: True, False except NameError: # Maintain compatibility with Python 2.2 True, False = 1, 0 import sys, os, getopt from spambayes import hammie, mboxutils from spambayes.Options import options program = sys.argv[0] TRAINED_HDR = "X-Spambayes-Trained" loud = True def msg_train(h, msg, is_spam, force): """Train bayes with a single message.""" # XXX: big hack -- why is email.Message unable to represent # multipart/alternative? try: msg.as_string() except TypeError: # We'll be unable to represent this as text :( return False if is_spam: spamtxt = options["Headers", "header_spam_string"] else: spamtxt = options["Headers", "header_ham_string"] oldtxt = msg.get(TRAINED_HDR) if force: # Train no matter what. if oldtxt != None: del msg[TRAINED_HDR] elif oldtxt == spamtxt: # Skip this one, we've already trained with it. return False elif oldtxt != None: # It's been trained, but as something else. Untrain. del msg[TRAINED_HDR] h.untrain(msg, not is_spam) h.train(msg, is_spam) msg.add_header(TRAINED_HDR, spamtxt) return True def maildir_train(h, path, is_spam, force, removetrained): """Train bayes with all messages from a maildir.""" if loud: print " Reading %s as Maildir" % (path,) import time import socket pid = os.getpid() host = socket.gethostname() counter = 0 trained = 0 for fn in os.listdir(path): cfn = os.path.join(path, fn) tfn = os.path.normpath(os.path.join(path, "..", "tmp", "%d.%d_%d.%s" % (time.time(), pid, counter, host))) if (os.path.isdir(cfn)): continue counter += 1 if loud: sys.stdout.write(" %s \r" % fn) sys.stdout.flush() f = file(cfn, "rb") msg = mboxutils.get_message(f) f.close() if not msg_train(h, msg, is_spam, force): continue trained += 1 f = file(tfn, "wb") f.write(msg.as_string()) f.close() # XXX: This will raise an exception on Windows. Do any Windows # people actually use Maildirs? os.rename(tfn, cfn) if (removetrained): os.unlink(cfn) if loud: print (" Trained %d out of %d messages " % (trained, counter)) def mbox_train(h, path, is_spam, force): """Train bayes with a Unix mbox""" if loud: print " Reading as Unix mbox" import mailbox import fcntl import tempfile # Open and lock the mailbox. Some systems require it be opened for # writes in order to assert an exclusive lock. f = file(path, "r+b") fcntl.flock(f, fcntl.LOCK_EX) mbox = mailbox.PortableUnixMailbox(f, mboxutils.get_message) outf = os.tmpfile() counter = 0 trained = 0 for msg in mbox: counter += 1 if loud: sys.stdout.write(" %s\r" % counter) sys.stdout.flush() if msg_train(h, msg, is_spam, force): trained += 1 # Write it out with the Unix "From " line outf.write(msg.as_string(True)) outf.seek(0) try: os.ftruncate(f.fileno(), 0) f.seek(0) except: # If anything goes wrong, don't try to write print "Problem truncating mbox--nothing written" raise try: for line in outf.xreadlines(): f.write(line) except: print >> sys.stderr ("Problem writing mbox! Sorry, " "I tried my best, but your mail " "may be corrupted.") raise fcntl.lockf(f, fcntl.LOCK_UN) f.close() if loud: print (" Trained %d out of %d messages " % (trained, counter)) def mhdir_train(h, path, is_spam, force): """Train bayes with an mh directory""" if loud: print " Reading as MH mailbox" import glob counter = 0 trained = 0 for fn in glob.glob(os.path.join(path, "[0-9]*")): counter += 1 cfn = fn tfn = os.path.join(path, "spambayes.tmp") if loud: sys.stdout.write(" %s \r" % fn) sys.stdout.flush() f = file(fn, "rb") msg = mboxutils.get_message(f) f.close() msg_train(h, msg, is_spam, force) trained += 1 f = file(tfn, "wb") f.write(msg.as_string()) f.close() # XXX: This will raise an exception on Windows. Do any Windows # people actually use MH directories? os.rename(tfn, cfn) if loud: print (" Trained %d out of %d messages " % (trained, counter)) def train(h, path, is_spam, force, trainnew, removetrained): if not os.path.exists(path): raise ValueError("Nonexistent path: %s" % path) elif os.path.isfile(path): mbox_train(h, path, is_spam, force) elif os.path.isdir(os.path.join(path, "cur")): maildir_train(h, os.path.join(path, "cur"), is_spam, force, removetrained) if trainnew: maildir_train(h, os.path.join(path, "new"), is_spam, force, removetrained) elif os.path.isdir(path): mhdir_train(h, path, is_spam, force) else: raise ValueError("Unable to determine mailbox type: " + path) def usage(code, msg=''): """Print usage message and sys.exit(code).""" if msg: print >> sys.stderr, msg print >> sys.stderr print >> sys.stderr, __doc__ % globals() sys.exit(code) def main(): """Main program; parse options and go.""" global loud try: opts, args = getopt.getopt(sys.argv[1:], 'hfqnrd:D:g:s:') except getopt.error, msg: usage(2, msg) if not opts: usage(2, "No options given") pck = None usedb = None force = False trainnew = False removetrained = False good = [] spam = [] for opt, arg in opts: if opt == '-h': usage(0) elif opt == "-f": force = True elif opt == "-n": trainnew = True elif opt == "-q": loud = False elif opt == '-g': good.append(arg) elif opt == '-s': spam.append(arg) elif opt == "-r": removetrained = True elif opt == "-d": usedb = True pck = arg elif opt == "-D": usedb = False pck = arg if args: usage(2, "Positional arguments not allowed") if usedb == None: usage(2, "Must specify one of -d or -D") h = hammie.open(pck, usedb, "c") for g in good: if loud: print "Training ham (%s):" % g train(h, g, False, force, trainnew, removetrained) save = True for s in spam: if loud: print "Training spam (%s):" % s train(h, s, True, force, trainnew, removetrained) save = True if save: h.store() if __name__ == "__main__": main() --- NEW FILE: sb-notesfilter.py --- #! /usr/bin/env python '''notesfilter.py - Lotus Notes Spambayes interface. Classes: Abstract: This module uses Spambayes as a filter against a Lotus Notes mail database. The Notes client must be running when this process is executed. It requires a Notes folder, named as a parameter, with four subfolders: Spam Ham Train as Spam Train as Ham Depending on the execution parameters, it will do any or all of the following steps, in the order given. 1. Train Spam from the Train as Spam folder (-t option) 2. Train Ham from the Train as Ham folder (-t option) 3. Replicate (-r option) 4. Classify the inbox (-c option) Mail that is to be trained as spam should be manually moved to that folder by the user. Likewise mail that is to be trained as ham. After training, spam is moved to the Spam folder and ham is moved to the Ham folder. Replication takes place if a remote server has been specified. This step may take a long time, depending on replication parameters and how much information there is to download, as well as line speed and server load. Please be patient if you run with replication. There is currently no progress bar or anything like that to tell you that it's working, but it is and will complete eventually. There is also no mechanism for notifying you that the replication failed. If it did, there is no harm done, and the program will continue execution. Mail that is classified as Spam is moved from the inbox to the Train as Spam folder. You should occasionally review your Spam folder for Ham that has mistakenly been classified as Spam. If there is any there, move it to the Train as Ham folder, so Spambayes will be less likely to make this mistake again. Mail that is classified as Ham or Unsure is left in the inbox. There is currently no means of telling if a mail was classified as Ham or Unsure. You should occasionally select some Ham and move it to the Train as Ham folder, so Spambayes can tell the difference between Spam and Ham. The goal is to maintain a relative balance between the number of Spam and the number of Ham that have been trained into the database. These numbers are reported every time this program executes. However, if the amount of Spam you receive far exceeds the amount of Ham you receive, it may be very difficult to maintain this balance. This is not a matter of great concern. Spambayes will still make very few mistakes in this circumstance. But, if this is the case, you should review your Spam folder for falsely classified Ham, and retrain those that you find, on a regular basis. This will prevent statistical error accumulation, which if allowed to continue, would cause Spambayes to tend to classify everything as Spam. Because there is no programmatic way to determine if a particular mail has been previously processed by this classification program, it keeps a pickled dictionary of notes mail ids, so that once a mail has been classified, it will not be classified again. The non-existence of is index file, named .sbindex, indicates to the system that this is an initialization execution. Rather than classify the inbox in this case, the contents of the inbox are placed in the index to note the 'starting point' of the system. After that, any new messages in the inbox are eligible for classification. Usage: notesfilter [options] note: option values with spaces in them must be enclosed in double quotes options: -d dbname : pickled training database filename -D dbname : dbm training database filename -l dbname : database filename of local mail replica e.g. localmail.nsf -r server : server address of the server mail database e.g. d27ml602/27/M/IBM if specified, will initiate a replication -f folder : Name of spambayes folder must have subfolders: Spam Ham Train as Spam Train as Ham -t : train contents of Train as Spam and Train as Ham -c : classify inbox -h : help -p : prompt "Press Enter to end" before ending This is useful for automated executions where the statistics output would otherwise be lost when the window closes. Examples: Replicate and classify inbox notesfilter -c -d notesbayes -r mynoteserv -l mail.nsf -f Spambayes Train Spam and Ham, then classify inbox notesfilter -t -c -d notesbayes -l mail.nsf -f Spambayes Replicate, then classify inbox notesfilter -c -d test7 -l mail.nsf -r nynoteserv -f Spambayes To Do: o Dump/purge notesindex file o Create correct folders if they do not exist o Options for some of this stuff? o pop3proxy style training/configuration interface? o parameter to retrain? o Suggestions? ''' # This module is part of the spambayes project, which is Copyright 2002 # The Python Software Foundation and is covered by the Python Software # Foundation license. __author__ = "Tim Stone " __credits__ = "Mark Hammond, for his remarkable win32 modules." from __future__ import generators try: True, False except NameError: # Maintain compatibility with Python 2.2 True, False = 1, 0 def bool(val): return not not val import sys from spambayes import tokenizer, storage from spambayes.Options import options import cPickle as pickle import errno import win32com.client import pywintypes import getopt def classifyInbox(v, vmoveto, bayes, ldbname, notesindex): # the notesindex hash ensures that a message is looked at only once if len(notesindex.keys()) == 0: firsttime = 1 else: firsttime = 0 docstomove = [] numham = 0 numspam = 0 numuns = 0 numdocs = 0 doc = v.GetFirstDocument() while doc: nid = doc.NOTEID if firsttime: notesindex[nid] = 'never classified' else: if not notesindex.has_key(nid): numdocs += 1 # Notes returns strings in unicode, and the Python # uni-decoder has trouble with these strings when # you try to print them. So don't... # The com interface returns basic data types as tuples # only, thus the subscript on GetItemValue try: subj = doc.GetItemValue('Subject')[0] except: subj = 'No Subject' try: body = doc.GetItemValue('Body')[0] except: body = 'No Body' message = "Subject: %s\r\n\r\n%s" % (subj, body) # generate_long_skips = True blows up on occasion, # probably due to this unicode problem. options["Tokenizer", "generate_long_skips"] = False tokens = tokenizer.tokenize(message) prob, clues = bayes.spamprob(tokens, evidence=True) if prob < options["Categorization", "ham_cutoff"]: disposition = options["Hammie", "header_ham_string"] numham += 1 elif prob > options["Categorization", "spam_cutoff"]: disposition = options["Hammie", "header_spam_string"] docstomove += [doc] numspam += 1 else: disposition = options["Hammie", "header_unsure_string"] numuns += 1 notesindex[nid] = 'classified' try: print "%s spamprob is %s" % (subj[:30], prob) except UnicodeError: print " spamprob is %s" % (prob) doc = v.GetNextDocument(doc) # docstomove list is built because moving documents in the middle of # the classification loop looses the iterator position for doc in docstomove: doc.RemoveFromFolder(v.Name) doc.PutInFolder(vmoveto.Name) print "%s documents processed" % (numdocs) print " %s classified as spam" % (numspam) print " %s classified as ham" % (numham) print " %s classified as unsure" % (numuns) def processAndTrain(v, vmoveto, bayes, is_spam, notesindex): if is_spam: str = options["Hammie", "header_spam_string"] else: str = options["Hammie", "header_ham_string"] print "Training %s" % (str) docstomove = [] doc = v.GetFirstDocument() while doc: try: subj = doc.GetItemValue('Subject')[0] except: subj = 'No Subject' try: body = doc.GetItemValue('Body')[0] except: body = 'No Body' message = "Subject: %s\r\n%s" % (subj, body) options["Tokenizer", "generate_long_skips"] = False tokens = tokenizer.tokenize(message) nid = doc.NOTEID if notesindex.has_key(nid): trainedas = notesindex[nid] if trainedas == options["Hammie", "header_spam_string"] and \ not is_spam: # msg is trained as spam, is to be retrained as ham bayes.unlearn(tokens, True) elif trainedas == options["Hammie", "header_ham_string"] and \ is_spam: # msg is trained as ham, is to be retrained as spam bayes.unlearn(tokens, False) bayes.learn(tokens, is_spam) notesindex[nid] = str docstomove += [doc] doc = v.GetNextDocument(doc) for doc in docstomove: doc.RemoveFromFolder(v.Name) doc.PutInFolder(vmoveto.Name) print "%s documents trained" % (len(docstomove)) def run(bdbname, useDBM, ldbname, rdbname, foldname, doTrain, doClassify): if useDBM: bayes = storage.DBDictClassifier(bdbname) else: bayes = storage.PickledClassifier(bdbname) try: fp = open("%s.sbindex" % (ldbname), 'rb') except IOError, e: if e.errno != errno.ENOENT: raise notesindex = {} print "%s.sbindex file not found, this is a first time run" \ % (ldbname) print "No classification will be performed" else: notesindex = pickle.load(fp) fp.close() sess = win32com.client.Dispatch("Lotus.NotesSession") try: sess.initialize() except pywintypes.com_error: print "Session aborted" sys.exit() db = sess.GetDatabase("",ldbname) vinbox = db.getView('($Inbox)') vspam = db.getView("%s\Spam" % (foldname)) vham = db.getView("%s\Ham" % (foldname)) vtrainspam = db.getView("%s\Train as Spam" % (foldname)) vtrainham = db.getView("%s\Train as Ham" % (foldname)) if doTrain: processAndTrain(vtrainspam, vspam, bayes, True, notesindex) # for some reason, using inbox as a target here loses the mail processAndTrain(vtrainham, vham, bayes, False, notesindex) if rdbname: print "Replicating..." db.Replicate(rdbname) print "Done" if doClassify: classifyInbox(vinbox, vtrainspam, bayes, ldbname, notesindex) print "The Spambayes database currently has %s Spam and %s Ham" \ % (bayes.nspam, bayes.nham) bayes.store() fp = open("%s.sbindex" % (ldbname), 'wb') pickle.dump(notesindex, fp) fp.close() if __name__ == '__main__': try: opts, args = getopt.getopt(sys.argv[1:], 'htcpd:D:l:r:f:') except getopt.error, msg: print >>sys.stderr, str(msg) + '\n\n' + __doc__ sys.exit() bdbname = None # bayes database name ldbname = None # local notes database name rdbname = None # remote notes database location sbfname = None # spambayes folder name doTrain = False doClassify = False doPrompt = False for opt, arg in opts: if opt == '-h': print >>sys.stderr, __doc__ sys.exit() elif opt == '-d': useDBM = False bdbname = arg elif opt == '-D': useDBM = True bdbname = arg elif opt == '-l': ldbname = arg elif opt == '-r': rdbname = arg elif opt == '-f': sbfname = arg elif opt == '-t': doTrain = True elif opt == '-c': doClassify = True elif opt == '-p': doPrompt = True if (bdbname and ldbname and sbfname and (doTrain or doClassify)): run(bdbname, useDBM, ldbname, rdbname, \ sbfname, doTrain, doClassify) if doPrompt: try: key = input("Press Enter to end") except SyntaxError: pass else: print >>sys.stderr, __doc__ --- NEW FILE: sb-pop3dnd.py --- #!/usr/bin/env python from __future__ import generators """ Overkill (someone *please* come up with something to call this script!) This application is a twisted cross between a POP3 proxy and an IMAP server. It sits between your mail client and your POP3 server (like any other POP3 proxy). While messages classified as ham are simply passed through the proxy, messages that are classified as spam or unsure are intercepted and passed to the IMAP server. The IMAP server offers three folders - one where messages classified as spam end up, one for messages it is unsure about, and one for training ham. In other words, to use this application, setup your mail client to connect to localhost, rather than directly to your POP3 server. Additionally, add a new IMAP account, also connecting to localhost. Setup the application via the web interface, and you are ready to go. Good messages will appear as per normal, but you will also have two new incoming folders, one for spam and one for ham. To train SpamBayes, use the spam folder, and the 'train_as_ham' folder. Any messages in these folders will be trained appropriately. This means that all messages that SpamBayes classifies as spam will also be trained as such. If you receive any 'false positives' (ham classified as spam), you *must* copy the message into the 'train_as_ham' folder to correct the training. You may also place any saved spam messages you have into this folder. So that SpamBayes knows about ham as well as spam, you will also need to move or copy mail into the 'train_as_ham' folder. These may come from the unsure folder, or from any other mail you have saved. It is a good idea to leave messages in the 'train_as_ham' and 'spam' folders, so that you can retrain from scratch if required. (However, you should always clear out your unsure folder, preferably moving or copying the messages into the appropriate training folder). This SpamBayes application is designed to work with Outlook Express, and provide the same sort of ease of use as the Outlook plugin. Although the majority of development and testing has been done with Outlook Express, any mail client that supports both IMAP and POP3 should be able to use this application - if the client enables the user to work with an IMAP account and POP3 account side-by-side (and move messages between them), then it should work equally as well as Outlook Express. This module includes the following classes: o IMAPFileMessage o IMAPFileMessageFactory o IMAPMailbox o SpambayesMailbox o Trainer o SpambayesAccount o SpambayesIMAPServer o OneParameterFactory o MyBayesProxy o MyBayesProxyListener o IMAPState """ todo = """ o Message flags are currently not persisted, but should be. The IMAPFileMessage class should be extended to do this. The same goes for the 'internaldate' of the message. o The RECENT flag should be unset at some point, but when? The RFC says that a message is recent if this is the first session to be notified about the message. Perhaps this can be done simply by *not* persisting this flag - i.e. the flag is always loaded as not recent, and only new messages are recent. The RFC says that if it is not possible to determine, then all messages should be recent, and this is what we currently do. o The Mailbox should be calling the appropriate listener functions (currently only newMessages is called on addMessage). flagsChanged should also be called on store, addMessage, or ??? o We cannot currently get part of a message via the BODY calls (with the <> operands), or get a part of a MIME message (by prepending a number). This should be added! o If the user clicks the 'save and shutdown' button on the web interface, this will only kill the POP3 proxy and web interface threads, and not the IMAP server. We need to monitor the thread that we kick off, and if it dies, we should die too. Need to figure out how to do this in twisted. o Suggestions? """ # This module is part of the spambayes project, which is Copyright 2002-3 # The Python Software Foundation and is covered by the Python Software # Foundation license. __author__ = "Tony Meyer " __credits__ = "All the Spambayes folk." try: True, False except NameError: # Maintain compatibility with Python 2.2 True, False = 1, 0 import os import re import sys import md5 import time import errno import types import thread import getopt import imaplib import operator import StringIO import email.Utils from twisted import cred from twisted.internet import defer from twisted.internet import reactor from twisted.internet.app import Application from twisted.internet.defer import maybeDeferred from twisted.internet.protocol import ServerFactory from twisted.protocols.imap4 import parseNestedParens, parseIdList from twisted.protocols.imap4 import IllegalClientResponse, IAccount from twisted.protocols.imap4 import collapseNestedLists, MessageSet from twisted.protocols.imap4 import IMAP4Server, MemoryAccount, IMailbox from twisted.protocols.imap4 import IMailboxListener, collapseNestedLists # Provide for those that don't have spambayes on their PYTHONPATH sys.path.insert(-1, os.path.dirname(os.getcwd())) from spambayes.Options import options from spambayes.message import Message from spambayes.tokenizer import tokenize from spambayes import FileCorpus, Dibbler from spambayes.Version import get_version_string from spambayes.ServerUI import ServerUserInterface from spambayes.UserInterface import UserInterfaceServer from pop3proxy import POP3ProxyBase, State, _addressPortStr, _recreateState def ensureDir(dirname): """Ensure that the given directory exists - in other words, if it does not exist, attempt to create it.""" try: os.mkdir(dirname) if options["globals", "verbose"]: print "Creating directory", dirname except OSError, e: if e.errno != errno.EEXIST: raise class IMAPFileMessage(FileCorpus.FileMessage): '''IMAP Message that persists as a file system artifact.''' def __init__(self, file_name, directory): """Constructor(message file name, corpus directory name).""" FileCorpus.FileMessage.__init__(self, file_name, directory) self.id = file_name self.directory = directory self.date = imaplib.Time2Internaldate(time.time())[1:-1] self.clear_flags() # IMessage implementation def getHeaders(self, negate, names): """Retrieve a group of message headers.""" headers = {} if not isinstance(names, tuple): names = (names,) for header, value in self.items(): if (header.upper() in names and not negate) or names == (): headers[header.upper()] = value return headers def getFlags(self): """Retrieve the flags associated with this message.""" return self._flags_iter() def _flags_iter(self): if self.deleted: yield "\\DELETED" if self.answered: yield "\\ANSWERED" if self.flagged: yield "\\FLAGGED" if self.seen: yield "\\SEEN" if self.draft: yield "\\DRAFT" if self.draft: yield "\\RECENT" def getInternalDate(self): """Retrieve the date internally associated with this message.""" return self.date def getBodyFile(self): """Retrieve a file object containing the body of this message.""" # Note only body, not headers! s = StringIO.StringIO() s.write(self.body()) s.seek(0) return s #return file(os.path.join(self.directory, self.id), "r") def getSize(self): """Retrieve the total size, in octets, of this message.""" return len(self.as_string()) def getUID(self): """Retrieve the unique identifier associated with this message.""" return self.id def getSubPart(self, part): """Retrieve a MIME sub-message @type part: C{int} @param part: The number of the part to retrieve, indexed from 0. @rtype: Any object implementing C{IMessage}. @return: The specified sub-part. """ # IMessage implementation ends def clear_flags(self): """Set all message flags to false.""" self.deleted = False self.answered = False self.flagged = False self.seen = False self.draft = False self.recent = False def set_flag(self, flag, value): # invalid flags are ignored flag = flag.upper() if flag == "\\DELETED": self.deleted = value elif flag == "\\ANSWERED": self.answered = value elif flag == "\\FLAGGED": self.flagged = value elif flag == "\\SEEN": self.seen = value elif flag == "\\DRAFT": self.draft = value else: print "Tried to set invalid flag", flag, "to", value def flags(self): """Return the message flags.""" all_flags = [] if self.deleted: all_flags.append("\\DELETED") if self.answered: all_flags.append("\\ANSWERED") if self.flagged: all_flags.append("\\FLAGGED") if self.seen: all_flags.append("\\SEEN") if self.draft: all_flags.append("\\DRAFT") if self.draft: all_flags.append("\\RECENT") return all_flags def train(self, classifier, isSpam): if self.GetTrained() == (not isSpam): classifier.unlearn(self.asTokens(), not isSpam) self.RememberTrained(None) if self.GetTrained() is None: classifier.learn(self.asTokens(), isSpam) self.RememberTrained(isSpam) classifier.store() def structure(self, ext=False): """Body structure data describes the MIME-IMB format of a message and consists of a sequence of mime type, mime subtype, parameters, content id, description, encoding, and size. The fields following the size field are variable: if the mime type/subtype is message/rfc822, the contained message's envelope information, body structure data, and number of lines of text; if the mime type is text, the number of lines of text. Extension fields may also be included; if present, they are: the MD5 hash of the body, body disposition, body language.""" s = [] for part in self.walk(): if part.get_content_charset() is not None: charset = ("charset", part.get_content_charset()) else: charset = None part_s = [part.get_main_type(), part.get_subtype(), charset, part.get('Content-Id'), part.get('Content-Description'), part.get('Content-Transfer-Encoding'), str(len(part.as_string()))] #if part.get_type() == "message/rfc822": # part_s.extend([envelope, body_structure_data, # part.as_string().count("\n")]) #elif part.get_main_type() == "text": if part.get_main_type() == "text": part_s.append(str(part.as_string().count("\n"))) if ext: part_s.extend([md5.new(part.as_string()).digest(), part.get('Content-Disposition'), part.get('Content-Language')]) s.append(part_s) if len(s) == 1: return s[0] return s def body(self): rfc822 = self.as_string() bodyRE = re.compile(r"\r?\n(\r?\n)(.*)", re.DOTALL + re.MULTILINE) bmatch = bodyRE.search(rfc822) return bmatch.group(2) def headers(self): rfc822 = self.as_string() bodyRE = re.compile(r"\r?\n(\r?\n)(.*)", re.DOTALL + re.MULTILINE) bmatch = bodyRE.search(rfc822) return rfc822[:bmatch.start(2)] def on(self, date1, date2): "contained within the date" raise NotImplementedError def before(self, date1, date2): "before the date" raise NotImplementedError def since(self, date1, date2): "within or after the date" raise NotImplementedError def string_contains(self, whole, sub): return whole.find(sub) != -1 def matches(self, criteria): """Return True iff the messages matches the specified IMAP criteria.""" match_tests = {"ALL" : [(True, True)], "ANSWERED" : [(self.answered, True)], "DELETED" : [(self.deleted, True)], "DRAFT" : [(self.draft, True)], "FLAGGED" : [(self.flagged, True)], "NEW" : [(self.recent, True), (self.seen, False)], "RECENT" : [(self.recent, True)], "SEEN" : [(self.seen, True)], "UNANSWERED" : [(self.answered, False)], "UNDELETED" : [(self.deleted, False)], "UNDRAFT" : [(self.draft, False)], "UNFLAGGED" : [(self.flagged, False)], "UNSEEN" : [(self.seen, False)], "OLD" : [(self.recent, False)], } complex_tests = {"BCC" : (self.string_contains, self.get("Bcc")), "SUBJECT" : (self.string_contains, self.get("Subject")), "CC" : (self.string_contains, self.get("Cc")), "BODY" : (self.string_contains, self.body()), "TO" : (self.string_contains, self.get("To")), "TEXT" : (self.string_contains, self.as_string()), "FROM" : (self.string_contains, self.get("From")), "SMALLER" : (operator.lt, len(self.as_string())), "LARGER" : (operator.gt, len(self.as_string())), "BEFORE" : (self.before, self.date), "ON" : (self.on, self.date), "SENTBEFORE" : (self.before, self.get("Date")), "SENTON" : (self.on, self.get("Date")), "SENTSINCE" : (self.since, self.get("Date")), "SINCE" : (self.since, self.date), } result = True test = None header = None header_field = None for c in criteria: if match_tests.has_key(c) and test is None and header is None: for (test, result) in match_tests[c]: result = result and (test == result) elif complex_tests.has_key(c) and test is None and header is None: test = complex_tests[c] elif test is not None and header is None: result = result and test[0](test[1], c) test = None elif c == "HEADER" and test is None: # the only criteria that uses the next _two_ elements header = c elif test is None and header is not None and header_field is None: header_field = c elif header is not None and header_field is not None and test is None: result = result and self.string_contains(self.get(header_field), c) header = None header_field = None return result """ Still to do: Messages with message sequence numbers corresponding to the specified message sequence number set UID Messages with unique identifiers corresponding to the specified unique identifier set. KEYWORD Messages with the specified keyword set. UNKEYWORD Messages that do not have the specified keyword set. NOT Messages that do not match the specified search key. OR Messages that match either search key. """ class IMAPFileMessageFactory(FileCorpus.FileMessageFactory): '''MessageFactory for IMAPFileMessage objects''' def create(self, key, directory): '''Create a message object from a filename in a directory''' return IMAPFileMessage(key, directory) class IMAPMailbox(cred.perspective.Perspective): __implements__ = (IMailbox,) def __init__(self, name, identity_name, id): cred.perspective.Perspective.__init__(self, name, identity_name) self.UID_validity = id self.listeners = [] def getUIDValidity(self): """Return the unique validity identifier for this mailbox.""" return self.UID_validity def addListener(self, listener): """Add a mailbox change listener.""" self.listeners.append(listener) def removeListener(self, listener): """Remove a mailbox change listener.""" self.listeners.remove(listener) class SpambayesMailbox(IMAPMailbox): def __init__(self, name, id, directory): IMAPMailbox.__init__(self, name, "spambayes", id) self.UID_validity = id ensureDir(directory) self.storage = FileCorpus.FileCorpus(IMAPFileMessageFactory(), directory, r"[0123456789]*") # UIDs are required to be strictly ascending. if len(self.storage.keys()) == 0: self.nextUID = 0 else: self.nextUID = long(self.storage.keys()[-1]) + 1 # Calculate initial recent and unseen counts # XXX Note that this will always end up with zero counts # XXX until the flags are persisted. self.unseen_count = 0 self.recent_count = 0 for msg in self.storage: if not msg.seen: self.unseen_count += 1 if msg.recent: self.recent_count += 1 def getUIDNext(self, increase=False): """Return the likely UID for the next message added to this mailbox.""" reply = str(self.nextUID) if increase: self.nextUID += 1 return reply def getUID(self, message): """Return the UID of a message in the mailbox.""" # Note that IMAP messages are 1-based, our messages are 0-based d = self.storage return long(d.keys()[message - 1]) def getFlags(self): """Return the flags defined in this mailbox.""" return ["\\Answered", "\\Flagged", "\\Deleted", "\\Seen", "\\Draft"] def getMessageCount(self): """Return the number of messages in this mailbox.""" return len(self.storage.keys()) def getRecentCount(self): """Return the number of messages with the 'Recent' flag.""" return self.recent_count def getUnseenCount(self): """Return the number of messages with the 'Unseen' flag.""" return self.unseen_count def isWriteable(self): """Get the read/write status of the mailbox.""" return True def destroy(self): """Called before this mailbox is deleted, permanently.""" # Our mailboxes cannot be deleted raise NotImplementedError def getHierarchicalDelimiter(self): """Get the character which delimits namespaces for in this mailbox.""" return '.' def requestStatus(self, names): """Return status information about this mailbox.""" answer = {} for request in names: request = request.upper() if request == "MESSAGES": answer[request] = self.getMessageCount() elif request == "RECENT": answer[request] = self.getRecentCount() elif request == "UIDNEXT": answer[request] = self.getUIDNext() elif request == "UIDVALIDITY": answer[request] = self.getUIDValidity() elif request == "UNSEEN": answer[request] = self.getUnseenCount() return answer def addMessage(self, message, flags=(), date=None): """Add the given message to this mailbox.""" msg = self.storage.makeMessage(self.getUIDNext(True)) msg.date = date msg.setPayload(message.read()) self.storage.addMessage(msg) self.store(MessageSet(long(msg.id), long(msg.id)), flags, 1, True) msg.recent = True msg.store() self.recent_count += 1 self.unseen_count += 1 for listener in self.listeners: listener.newMessages(self.getMessageCount(), self.getRecentCount()) d = defer.Deferred() reactor.callLater(0, d.callback, self.storage.keys().index(msg.id)) return d def expunge(self): """Remove all messages flagged \\Deleted.""" deleted_messages = [] for msg in self.storage: if msg.deleted: if not msg.seen: self.unseen_count -= 1 if msg.recent: self.recent_count -= 1 deleted_messages.append(long(msg.id)) self.storage.removeMessage(msg) if deleted_messages != []: for listener in self.listeners: listener.newMessages(self.getMessageCount(), self.getRecentCount()) return deleted_messages def search(self, query, uid): """Search for messages that meet the given query criteria. @type query: C{list} @param query: The search criteria @rtype: C{list} @return: A list of message sequence numbers or message UIDs which match the search criteria. """ if self.getMessageCount() == 0: return [] all_msgs = MessageSet(long(self.storage.keys()[0]), long(self.storage.keys()[-1])) matches = [] for id, msg in self._messagesIter(all_msgs, uid): for q in query: if msg.matches(q): matches.append(id) break return matches def _messagesIter(self, messages, uid): if uid: messages.last = long(self.storage.keys()[-1]) else: messages.last = self.getMessageCount() for id in messages: if uid: msg = self.storage.get(str(id)) else: msg = self.storage.get(str(self.getUID(id))) if msg is None: # Non-existant message. continue msg.load() yield (id, msg) def fetch(self, messages, uid): """Retrieve one or more messages.""" return self._messagesIter(messages, uid) def store(self, messages, flags, mode, uid): """Set the flags of one or more messages.""" stored_messages = {} for id, msg in self._messagesIter(messages, uid): if mode == 0: msg.clear_flags() value = True elif mode == -1: value = False elif mode == 1: value = True for flag in flags: if flag == '(' or flag == ')': continue if flag == "SEEN" and value == True and msg.seen == False: self.unseen_count -= 1 if flag == "SEEN" and value == False and msg.seen == True: self.unseen_count += 1 msg.set_flag(flag, value) stored_messages[id] = msg.flags() return stored_messages class Trainer(object): """Listens to a given mailbox and trains new messages as spam or ham.""" __implements__ = (IMailboxListener,) def __init__(self, mailbox, asSpam): self.mailbox = mailbox self.asSpam = asSpam def modeChanged(self, writeable): # We don't care pass def flagsChanged(self, newFlags): # We don't care pass def newMessages(self, exists, recent): # We don't get passed the actual message, or the id of # the message, of even the message number. We just get # the total number of new/recent messages. # However, this function should be called _every_ time # that a new message appears, so we should be able to # assume that the last message is the new one. # (We ignore the recent count) if exists is not None: id = self.mailbox.getUID(exists) msg = self.mailbox.storage[str(id)] msg.train(state.bayes, self.asSpam) class SpambayesAccount(MemoryAccount): """Account for Spambayes server.""" def __init__(self, id, ham, spam, unsure): MemoryAccount.__init__(self, id) self.mailboxes = {"SPAM" : spam, "UNSURE" : unsure, "TRAIN_AS_HAM" : ham} def select(self, name, readwrite=1): # 'INBOX' is a special case-insensitive name meaning the # primary mailbox for the user; we interpret this as an alias # for 'spam' if name.upper() == "INBOX": name = "SPAM" return MemoryAccount.select(self, name, readwrite) class SpambayesIMAPServer(IMAP4Server): IDENT = "Spambayes IMAP Server IMAP4rev1 Ready" def __init__(self, user_account): IMAP4Server.__init__(self) self.account = user_account def authenticateLogin(self, user, passwd): """Lookup the account associated with the given parameters.""" if user == options["imapserver", "username"] and \ passwd == options["imapserver", "password"]: return (IAccount, self.account, None) raise cred.error.UnauthorizedLogin() def connectionMade(self): state.activeIMAPSessions += 1 state.totalIMAPSessions += 1 IMAP4Server.connectionMade(self) def connectionLost(self, reason): state.activeIMAPSessions -= 1 IMAP4Server.connectionLost(self, reason) def do_CREATE(self, tag, args): """Creating new folders on the server is not permitted.""" self.sendNegativeResponse(tag, \ "Creation of new folders is not permitted") auth_CREATE = (do_CREATE, IMAP4Server.arg_astring) select_CREATE = auth_CREATE def do_DELETE(self, tag, args): """Deleting folders on the server is not permitted.""" self.sendNegativeResponse(tag, \ "Deletion of folders is not permitted") auth_DELETE = (do_DELETE, IMAP4Server.arg_astring) select_DELETE = auth_DELETE class OneParameterFactory(ServerFactory): """A factory that allows a single parameter to be passed to the created protocol.""" def buildProtocol(self, addr): """Create an instance of a subclass of Protocol, passing a single parameter.""" if self.parameter is not None: p = self.protocol(self.parameter) else: p = self.protocol() p.factory = self return p class MyBayesProxy(POP3ProxyBase): """Proxies between an email client and a POP3 server, redirecting mail to the imap server as necessary. It acts on the following POP3 commands: o RETR: o Adds the judgement header based on the raw headers and body of the message. """ intercept_message = 'From: "Spambayes" \n' \ 'Subject: Spambayes Intercept\n\nA message ' \ 'was intercepted by Spambayes (it scored %s).\n' \ '\nYou may find it in the Spam or Unsure ' \ 'folder.\n\n.\n' def __init__(self, clientSocket, serverName, serverPort, spam, unsure): POP3ProxyBase.__init__(self, clientSocket, serverName, serverPort) self.handlers = {'RETR': self.onRetr} state.totalSessions += 1 state.activeSessions += 1 self.isClosed = False self.spam_folder = spam self.unsure_folder = unsure def send(self, data): """Logs the data to the log file.""" if options["globals", "verbose"]: state.logFile.write(data) state.logFile.flush() try: return POP3ProxyBase.send(self, data) except socket.error: self.close() def recv(self, size): """Logs the data to the log file.""" data = POP3ProxyBase.recv(self, size) if options["globals", "verbose"]: state.logFile.write(data) state.logFile.flush() return data def close(self): # This can be called multiple times by async. if not self.isClosed: self.isClosed = True state.activeSessions -= 1 POP3ProxyBase.close(self) def onTransaction(self, command, args, response): """Takes the raw request and response, and returns the (possibly processed) response to pass back to the email client. """ handler = self.handlers.get(command, self.onUnknown) return handler(command, args, response) def onRetr(self, command, args, response): """Classifies the message. If the result is ham, then simply pass it through. If the result is an unsure or spam, move it to the appropriate IMAP folder.""" # Use '\n\r?\n' to detect the end of the headers in case of # broken emails that don't use the proper line separators. if re.search(r'\n\r?\n', response): # Break off the first line, which will be '+OK'. ok, messageText = response.split('\n', 1) prob = state.bayes.spamprob(tokenize(messageText)) if prob < options["Categorization", "ham_cutoff"]: # Return the +OK and the message with the header added. state.numHams += 1 return ok + "\n" + messageText elif prob > options["Categorization", "spam_cutoff"]: dest_folder = self.spam_folder state.numSpams += 1 else: dest_folder = self.unsure_folder state.numUnsure += 1 msg = StringIO.StringIO(messageText) date = imaplib.Time2Internaldate(time.time())[1:-1] dest_folder.addMessage(msg, (), date) # We have to return something, because the client is expecting # us to. We return a short message indicating that a message # was intercepted. return ok + "\n" + self.intercept_message % (prob,) else: # Must be an error response. return response def onUnknown(self, command, args, response): """Default handler; returns the server's response verbatim.""" return response class MyBayesProxyListener(Dibbler.Listener): """Listens for incoming email client connections and spins off MyBayesProxy objects to serve them. """ def __init__(self, serverName, serverPort, proxyPort, spam, unsure): proxyArgs = (serverName, serverPort, spam, unsure) Dibbler.Listener.__init__(self, proxyPort, MyBayesProxy, proxyArgs) print 'Listener on port %s is proxying %s:%d' % \ (_addressPortStr(proxyPort), serverName, serverPort) class IMAPState(State): def __init__(self): State.__init__(self) # Set up the extra statistics. self.totalIMAPSessions = 0 self.activeIMAPSessions = 0 def buildServerStrings(self): """After the server details have been set up, this creates string versions of the details, for display in the Status panel.""" self.serverPortString = str(self.imap_port) # Also build proxy strings State.buildServerStrings(self) state = IMAPState() # =================================================================== # __main__ driver. # =================================================================== def setup(): # Setup app, boxes, trainers and account proxyListeners = [] app = Application("SpambayesIMAPServer") spam_box = SpambayesMailbox("Spam", 0, options["imapserver", "spam_directory"]) unsure_box = SpambayesMailbox("Unsure", 1, options["imapserver", "unsure_directory"]) ham_train_box = SpambayesMailbox("TrainAsHam", 2, options["imapserver", "ham_directory"]) spam_trainer = Trainer(spam_box, True) ham_trainer = Trainer(ham_train_box, False) spam_box.addListener(spam_trainer) ham_train_box.addListener(ham_trainer) user_account = SpambayesAccount(options["imapserver", "username"], ham_train_box, spam_box, unsure_box) # add IMAP4 server f = OneParameterFactory() f.protocol = SpambayesIMAPServer f.parameter = user_account state.imap_port = options["imapserver", "port"] app.listenTCP(state.imap_port, f) # add POP3 proxy state.createWorkers() for (server, serverPort), proxyPort in zip(state.servers, state.proxyPorts): listener = MyBayesProxyListener(server, serverPort, proxyPort, spam_box, unsure_box) proxyListeners.append(listener) state.buildServerStrings() # add web interface httpServer = UserInterfaceServer(state.uiPort) serverUI = ServerUserInterface(state, _recreateState) httpServer.register(serverUI) return app def run(): # Read the arguments. try: opts, args = getopt.getopt(sys.argv[1:], 'hbd:D:u:') except getopt.error, msg: print >>sys.stderr, str(msg) + '\n\n' + __doc__ sys.exit() launchUI = False for opt, arg in opts: if opt == '-h': print >>sys.stderr, __doc__ sys.exit() elif opt == '-b': launchUI = True elif opt == '-d': # dbm file state.useDB = True options["Storage", "persistent_storage_file"] = arg elif opt == '-D': # pickle file state.useDB = False options["Storage", "persistent_storage_file"] = arg elif opt == '-u': state.uiPort = int(arg) # Let the user know what they are using... print get_version_string("IMAP Server") print "and engine %s.\n" % (get_version_string(),) # setup everything app = setup() # kick things off thread.start_new_thread(Dibbler.run, (launchUI,)) app.run(save=False) if __name__ == "__main__": run() --- NEW FILE: sb-server.py --- #!/usr/bin/env python """A POP3 proxy that works with classifier.py, and adds a simple X-Spambayes-Classification header (ham/spam/unsure) to each incoming email. You point pop3proxy at your POP3 server, and configure your email client to collect mail from the proxy then filter on the added header. Usage: pop3proxy.py [options] [ []] is the name of your real POP3 server is the port number of your real POP3 server, which defaults to 110. options: -h : Displays this help message. -d FILE : use the named DBM database file -D FILE : the the named Pickle database file -l port : proxy listens on this port number (default 110) -u port : User interface listens on this port number (default 8880; Browse http://localhost:8880/) -b : Launch a web browser showing the user interface. All command line arguments and switches take their default values from the [pop3proxy] and [html_ui] sections of bayescustomize.ini. For safety, and to help debugging, the whole POP3 conversation is written out to _pop3proxy.log for each run, if options["globals", "verbose"] is True. To make rebuilding the database easier, uploaded messages are appended to _pop3proxyham.mbox and _pop3proxyspam.mbox. """ # This module is part of the spambayes project, which is Copyright 2002 # The Python Software Foundation and is covered by the Python Software # Foundation license. __author__ = "Richie Hindle " __credits__ = "Tim Peters, Neale Pickett, Tim Stone, all the Spambayes folk." try: True, False except NameError: # Maintain compatibility with Python 2.2 True, False = 1, 0 todo = """ Web training interface: User interface improvements: o Once the pieces are on separate pages, make the paste box bigger. o Deployment: Windows executable? atlaxwin and ctypes? Or just webbrowser? o Save the stats (num classified, etc.) between sessions. o "Reload database" button. New features: o Online manual. o Links to project homepage, mailing list, etc. o List of words with stats (it would have to be paged!) a la SpamSieve. Code quality: o Cope with the email client timing out and closing the connection. Info: o Slightly-wordy index page; intro paragraph for each page. o In both stats and training results, report nham and nspam - warn if they're very different (for some value of 'very'). o "Links" section (on homepage?) to project homepage, mailing list, etc. Gimmicks: o Classify a web page given a URL. o Graphs. Of something. Who cares what? o NNTP proxy. o Zoe...! """ import os, sys, re, errno, getopt, time, traceback, socket, cStringIO from thread import start_new_thread from email.Header import Header import spambayes.message from spambayes import Dibbler from spambayes import storage from spambayes.FileCorpus import FileCorpus, ExpiryFileCorpus from spambayes.FileCorpus import FileMessageFactory, GzipFileMessageFactory from spambayes.Options import options from spambayes.UserInterface import UserInterfaceServer from spambayes.ProxyUI import ProxyUserInterface from spambayes.Version import get_version_string # Increase the stack size on MacOS X. Stolen from Lib/test/regrtest.py if sys.platform == 'darwin': try: import resource except ImportError: pass else: soft, hard = resource.getrlimit(resource.RLIMIT_STACK) newsoft = min(hard, max(soft, 1024*2048)) resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) # number to add to STAT length for each msg to fudge for spambayes headers HEADER_SIZE_FUDGE_FACTOR = 512 class ServerLineReader(Dibbler.BrighterAsyncChat): """An async socket that reads lines from a remote server and simply calls a callback with the data. The BayesProxy object can't connect to the real POP3 server and talk to it synchronously, because that would block the process.""" lineCallback = None def __init__(self, serverName, serverPort, lineCallback): Dibbler.BrighterAsyncChat.__init__(self) self.lineCallback = lineCallback self.request = '' self.set_terminator('\r\n') self.create_socket(socket.AF_INET, socket.SOCK_STREAM) try: self.connect((serverName, serverPort)) except socket.error, e: error = "Can't connect to %s:%d: %s" % (serverName, serverPort, e) print >>sys.stderr, error self.lineCallback('-ERR %s\r\n' % error) self.lineCallback('') # "The socket's been closed." self.close() def collect_incoming_data(self, data): self.request = self.request + data def found_terminator(self): self.lineCallback(self.request + '\r\n') self.request = '' def handle_close(self): self.lineCallback('') self.close() class POP3ProxyBase(Dibbler.BrighterAsyncChat): """An async dispatcher that understands POP3 and proxies to a POP3 server, calling `self.onTransaction(request, response)` for each transaction. Responses are not un-byte-stuffed before reaching self.onTransaction() (they probably should be for a totally generic POP3ProxyBase class, but BayesProxy doesn't need it and it would mean re-stuffing them afterwards). self.onTransaction() should return the response to pass back to the email client - the response can be the verbatim response or a processed version of it. The special command 'KILL' kills it (passing a 'QUIT' command to the server). """ def __init__(self, clientSocket, serverName, serverPort): Dibbler.BrighterAsyncChat.__init__(self, clientSocket) self.request = '' self.response = '' self.set_terminator('\r\n') self.command = '' # The POP3 command being processed... self.args = [] # ...and its arguments self.isClosing = False # Has the server closed the socket? self.seenAllHeaders = False # For the current RETR or TOP self.startTime = 0 # (ditto) self.serverSocket = ServerLineReader(serverName, serverPort, self.onServerLine) def onTransaction(self, command, args, response): """Overide this. Takes the raw request and the response, and returns the (possibly processed) response to pass back to the email client. """ raise NotImplementedError def onServerLine(self, line): """A line of response has been received from the POP3 server.""" isFirstLine = not self.response self.response = self.response + line # Is this the line that terminates a set of headers? self.seenAllHeaders = self.seenAllHeaders or line in ['\r\n', '\n'] # Has the server closed its end of the socket? if not line: self.isClosing = True # If we're not processing a command, just echo the response. if not self.command: self.push(self.response) self.response = '' # Time out after 30 seconds for message-retrieval commands if # all the headers are down. The rest of the message will proxy # straight through. if self.command in ['TOP', 'RETR'] and \ self.seenAllHeaders and time.time() > self.startTime + 30: self.onResponse() self.response = '' # If that's a complete response, handle it. elif not self.isMultiline() or line == '.\r\n' or \ (isFirstLine and line.startswith('-ERR')): self.onResponse() self.response = '' def isMultiline(self): """Returns True if the request should get a multiline response (assuming the response is positive). """ if self.command in ['USER', 'PASS', 'APOP', 'QUIT', 'STAT', 'DELE', 'NOOP', 'RSET', 'KILL']: return False elif self.command in ['RETR', 'TOP', 'CAPA']: return True elif self.command in ['LIST', 'UIDL']: return len(self.args) == 0 else: # Assume that an unknown command will get a single-line # response. This should work for errors and for POP-AUTH, # and is harmless even for multiline responses - the first # line will be passed to onTransaction and ignored, then the # rest will be proxied straight through. return False def collect_incoming_data(self, data): """Asynchat override.""" self.request = self.request + data def found_terminator(self): """Asynchat override.""" verb = self.request.strip().upper() if verb == 'KILL': self.socket.shutdown(2) self.close() raise SystemExit elif verb == 'CRASH': # For testing x = 0 y = 1/x self.serverSocket.push(self.request + '\r\n') if self.request.strip() == '': # Someone just hit the Enter key. self.command = '' self.args = [] else: # A proper command. splitCommand = self.request.strip().split() self.command = splitCommand[0].upper() self.args = splitCommand[1:] self.startTime = time.time() self.request = '' def onResponse(self): # We don't support pipelining, so if the command is CAPA and the # response includes PIPELINING, hack out that line of the response. if self.command == 'CAPA': pipelineRE = r'(?im)^PIPELINING[^\n]*\n' self.response = re.sub(pipelineRE, '', self.response) # Pass the request and the raw response to the subclass and # send back the cooked response. if self.response: cooked = self.onTransaction(self.command, self.args, self.response) self.push(cooked) # If onServerLine() decided that the server has closed its # socket, close this one when the response has been sent. if self.isClosing: self.close_when_done() # Reset. self.command = '' self.args = [] self.isClosing = False self.seenAllHeaders = False class BayesProxyListener(Dibbler.Listener): """Listens for incoming email client connections and spins off BayesProxy objects to serve them. """ def __init__(self, serverName, serverPort, proxyPort): proxyArgs = (serverName, serverPort) Dibbler.Listener.__init__(self, proxyPort, BayesProxy, proxyArgs) print 'Listener on port %s is proxying %s:%d' % \ (_addressPortStr(proxyPort), serverName, serverPort) class BayesProxy(POP3ProxyBase): """Proxies between an email client and a POP3 server, inserting judgement headers. It acts on the following POP3 commands: o STAT: o Adds the size of all the judgement headers to the maildrop size. o LIST: o With no message number: adds the size of an judgement header to the message size for each message in the scan listing. o With a message number: adds the size of an judgement header to the message size. o RETR: o Adds the judgement header based on the raw headers and body of the message. o TOP: o Adds the judgement header based on the raw headers and as much of the body as the TOP command retrieves. This can mean that the header might have a different value for different calls to TOP, or for calls to TOP vs. calls to RETR. I'm assuming that the email client will either not make multiple calls, or will cope with the headers being different. o USER: o Does no processing based on the USER command itself, but expires any old messages in the three caches. """ def __init__(self, clientSocket, serverName, serverPort): POP3ProxyBase.__init__(self, clientSocket, serverName, serverPort) self.handlers = {'STAT': self.onStat, 'LIST': self.onList, 'RETR': self.onRetr, 'TOP': self.onTop, 'USER': self.onUser} state.totalSessions += 1 state.activeSessions += 1 self.isClosed = False def send(self, data): """Logs the data to the log file.""" if options["globals", "verbose"]: state.logFile.write(data) state.logFile.flush() try: return POP3ProxyBase.send(self, data) except socket.error: # The email client has closed the connection - 40tude Dialog # does this immediately after issuing a QUIT command, # without waiting for the response. self.close() def recv(self, size): """Logs the data to the log file.""" data = POP3ProxyBase.recv(self, size) if options["globals", "verbose"]: state.logFile.write(data) state.logFile.flush() return data def close(self): # This can be called multiple times by async. if not self.isClosed: self.isClosed = True state.activeSessions -= 1 POP3ProxyBase.close(self) def onTransaction(self, command, args, response): """Takes the raw request and response, and returns the (possibly processed) response to pass back to the email client. """ handler = self.handlers.get(command, self.onUnknown) return handler(command, args, response) def onStat(self, command, args, response): """Adds the size of all the judgement headers to the maildrop size.""" match = re.search(r'^\+OK\s+(\d+)\s+(\d+)(.*)\r\n', response) if match: count = int(match.group(1)) size = int(match.group(2)) + HEADER_SIZE_FUDGE_FACTOR * count return '+OK %d %d%s\r\n' % (count, size, match.group(3)) else: return response def onList(self, command, args, response): """Adds the size of an judgement header to the message size(s).""" if response.count('\r\n') > 1: # Multiline: all lines but the first contain a message size. lines = response.split('\r\n') outputLines = [lines[0]] for line in lines[1:]: match = re.search(r'^(\d+)\s+(\d+)', line) if match: number = int(match.group(1)) size = int(match.group(2)) + HEADER_SIZE_FUDGE_FACTOR line = "%d %d" % (number, size) outputLines.append(line) return '\r\n'.join(outputLines) else: # Single line. match = re.search(r'^\+OK\s+(\d+)\s+(\d+)(.*)\r\n', response) if match: messageNumber = match.group(1) size = int(match.group(2)) + HEADER_SIZE_FUDGE_FACTOR trailer = match.group(3) return "+OK %s %s%s\r\n" % (messageNumber, size, trailer) else: return response def onRetr(self, command, args, response): """Adds the judgement header based on the raw headers and body of the message.""" # Use '\n\r?\n' to detect the end of the headers in case of # broken emails that don't use the proper line separators. if re.search(r'\n\r?\n', response): # Remove the trailing .\r\n before passing to the email parser. # Thanks to Scott Schlesier for this fix. terminatingDotPresent = (response[-4:] == '\n.\r\n') if terminatingDotPresent: response = response[:-3] # Break off the first line, which will be '+OK'. ok, messageText = response.split('\n', 1) try: msg = spambayes.message.SBHeaderMessage() msg.setPayload(messageText) msg.setId(state.getNewMessageName()) # Now find the spam disposition and add the header. (prob, clues) = state.bayes.spamprob(msg.asTokens(),\ evidence=True) msg.addSBHeaders(prob, clues) # Check for "RETR" or "TOP N 99999999" - fetchmail without # the 'fetchall' option uses the latter to retrieve messages. if (command == 'RETR' or (command == 'TOP' and len(args) == 2 and args[1] == '99999999')): cls = msg.GetClassification() if cls == options["Headers", "header_ham_string"]: state.numHams += 1 elif cls == options["Headers", "header_spam_string"]: state.numSpams += 1 else: state.numUnsure += 1 # Suppress caching of "Precedence: bulk" or # "Precedence: list" ham if the options say so. isSuppressedBulkHam = \ (cls == options["Headers", "header_ham_string"] and options["pop3proxy", "no_cache_bulk_ham"] and msg.get('precedence') in ['bulk', 'list']) # Suppress large messages if the options say so. size_limit = options["pop3proxy", "no_cache_large_messages"] isTooBig = size_limit > 0 and \ len(messageText) > size_limit # Cache the message. Don't pollute the cache with test # messages or suppressed bulk ham. if (not state.isTest and options["pop3proxy", "cache_messages"] and not isSuppressedBulkHam and not isTooBig): # Write the message into the Unknown cache. message = state.unknownCorpus.makeMessage(msg.getId()) message.setSubstance(msg.as_string()) state.unknownCorpus.addMessage(message) # We'll return the message with the headers added. We take # all the headers from the SBHeaderMessage, but take the body # directly from the POP3 conversation, because the # SBHeaderMessage might have "fixed" a partial message by # appending a closing boundary separator. Remember we can # be dealing with partial message here because of the timeout # code in onServerLine. headers = [] for name, value in msg.items(): header = "%s: %s" % (name, value) headers.append(re.sub(r'\r?\n', '\r\n', header)) body = re.split(r'\n\r?\n', messageText, 1)[1] messageText = "\r\n".join(headers) + "\r\n\r\n" + body except: # Something nasty happened while parsing or classifying - # report the exception in a hand-appended header and recover. # This is one case where an unqualified 'except' is OK, 'cos # anything's better than destroying people's email... stream = cStringIO.StringIO() traceback.print_exc(None, stream) details = stream.getvalue() # Build the header. This will strip leading whitespace from # the lines, so we add a leading dot to maintain indentation. detailLines = details.strip().split('\n') dottedDetails = '\n.'.join(detailLines) headerName = 'X-Spambayes-Exception' header = Header(dottedDetails, header_name=headerName) # Insert the header, converting email.Header's '\n' line # breaks to POP3's '\r\n'. headers, body = re.split(r'\n\r?\n', messageText, 1) header = re.sub(r'\r?\n', '\r\n', str(header)) headers += "\n%s: %s\r\n\r\n" % (headerName, header) messageText = headers + body # Print the exception and a traceback. print >>sys.stderr, details # Restore the +OK and the POP3 .\r\n terminator if there was one. retval = ok + "\n" + messageText if terminatingDotPresent: retval += '.\r\n' return retval else: # Must be an error response. return response def onTop(self, command, args, response): """Adds the judgement header based on the raw headers and as much of the body as the TOP command retrieves.""" # Easy (but see the caveat in BayesProxy.__doc__). return self.onRetr(command, args, response) def onUser(self, command, args, response): """Spins off three separate threads that expires any old messages in the three caches, but does not do any processing of the USER command itself.""" start_new_thread(state.spamCorpus.removeExpiredMessages, ()) start_new_thread(state.hamCorpus.removeExpiredMessages, ()) start_new_thread(state.unknownCorpus.removeExpiredMessages, ()) return response def onUnknown(self, command, args, response): """Default handler; returns the server's response verbatim.""" return response # This keeps the global state of the module - the command-line options, # statistics like how many mails have been classified, the handle of the # log file, the Classifier and FileCorpus objects, and so on. class State: def __init__(self): """Initialises the State object that holds the state of the app. The default settings are read from Options.py and bayescustomize.ini and are then overridden by the command-line processing code in the __main__ code below.""" # Open the log file. if options["globals", "verbose"]: self.logFile = open('_pop3proxy.log', 'wb', 0) self.servers = [] self.proxyPorts = [] if options["pop3proxy", "servers"]: for server in options["pop3proxy", "servers"]: server = server.strip() if server.find(':') > -1: server, port = server.split(':', 1) else: port = '110' self.servers.append((server, int(port))) if options["pop3proxy", "ports"]: splitPorts = options["pop3proxy", "ports"] self.proxyPorts = map(_addressAndPort, splitPorts) if len(self.servers) != len(self.proxyPorts): print "pop3proxy_servers & pop3proxy_ports are different lengths!" sys.exit() # Load up the other settings from Option.py / bayescustomize.ini self.useDB = options["pop3proxy", "persistent_use_database"] self.uiPort = options["html_ui", "port"] self.launchUI = options["html_ui", "launch_browser"] self.gzipCache = options["pop3proxy", "cache_use_gzip"] self.cacheExpiryDays = options["pop3proxy", "cache_expiry_days"] self.runTestServer = False self.isTest = False # Set up the statistics. self.totalSessions = 0 self.activeSessions = 0 self.numSpams = 0 self.numHams = 0 self.numUnsure = 0 # Unique names for cached messages - see `getNewMessageName()` below. self.lastBaseMessageName = '' self.uniquifier = 2 def buildServerStrings(self): """After the server details have been set up, this creates string versions of the details, for display in the Status panel.""" serverStrings = ["%s:%s" % (s, p) for s, p in self.servers] self.serversString = ', '.join(serverStrings) self.proxyPortsString = ', '.join(map(_addressPortStr, self.proxyPorts)) def createWorkers(self): """Using the options that were initialised in __init__ and then possibly overridden by the driver code, create the Bayes object, the Corpuses, the Trainers and so on.""" print "Loading database...", if self.isTest: self.useDB = True options["pop3proxy", "persistent_storage_file"] = \ '_pop3proxy_test.pickle' # This is never saved. filename = options["Storage", "persistent_storage_file"] filename = os.path.expanduser(filename) self.bayes = storage.open_storage(filename, self.useDB) # Don't set up the caches and training objects when running the self-test, # so as not to clutter the filesystem. if not self.isTest: def ensureDir(dirname): try: os.mkdir(dirname) except OSError, e: if e.errno != errno.EEXIST: raise # Create/open the Corpuses. Use small cache sizes to avoid hogging # lots of memory. map(ensureDir, [options["pop3proxy", "spam_cache"], options["pop3proxy", "ham_cache"], options["pop3proxy", "unknown_cache"]]) if self.gzipCache: factory = GzipFileMessageFactory() else: factory = FileMessageFactory() age = options["pop3proxy", "cache_expiry_days"]*24*60*60 self.spamCorpus = ExpiryFileCorpus(age, factory, options["pop3proxy", "spam_cache"], '[0123456789\-]*', cacheSize=20) self.hamCorpus = ExpiryFileCorpus(age, factory, options["pop3proxy", "ham_cache"], '[0123456789\-]*', cacheSize=20) self.unknownCorpus = ExpiryFileCorpus(age, factory, options["pop3proxy", "unknown_cache"], '[0123456789\-]*', cacheSize=20) # Given that (hopefully) users will get to the stage # where they do not need to do any more regular training to # be satisfied with spambayes' performance, we expire old # messages from not only the trained corpora, but the unknown # as well. self.spamCorpus.removeExpiredMessages() self.hamCorpus.removeExpiredMessages() self.unknownCorpus.removeExpiredMessages() # Create the Trainers. self.spamTrainer = storage.SpamTrainer(self.bayes) self.hamTrainer = storage.HamTrainer(self.bayes) self.spamCorpus.addObserver(self.spamTrainer) self.hamCorpus.addObserver(self.hamTrainer) def getNewMessageName(self): # The message name is the time it arrived, with a uniquifier # appended if two arrive within one clock tick of each other. messageName = "%10.10d" % long(time.time()) if messageName == self.lastBaseMessageName: messageName = "%s-%d" % (messageName, self.uniquifier) self.uniquifier += 1 else: self.lastBaseMessageName = messageName self.uniquifier = 2 return messageName # Option-parsing helper functions def _addressAndPort(s): """Decode a string representing a port to bind to, with optional address.""" s = s.strip() if ':' in s: addr, port = s.split(':') return addr, int(port) else: return '', int(s) def _addressPortStr((addr, port)): """Encode a string representing a port to bind to, with optional address.""" if not addr: return str(port) else: return '%s:%d' % (addr, port) state = State() proxyListeners = [] def _createProxies(servers, proxyPorts): """Create BayesProxyListeners for all the given servers.""" for (server, serverPort), proxyPort in zip(servers, proxyPorts): listener = BayesProxyListener(server, serverPort, proxyPort) proxyListeners.append(listener) def _recreateState(): global state state = State() # Close the existing listeners and create new ones. This won't # affect any running proxies - once a listener has created a proxy, # that proxy is then independent of it. for proxy in proxyListeners: proxy.close() del proxyListeners[:] prepare(state) _createProxies(state.servers, state.proxyPorts) return state def main(servers, proxyPorts, uiPort, launchUI): """Runs the proxy forever or until a 'KILL' command is received or someone hits Ctrl+Break.""" _createProxies(servers, proxyPorts) httpServer = UserInterfaceServer(uiPort) proxyUI = ProxyUserInterface(state, _recreateState) httpServer.register(proxyUI) Dibbler.run(launchBrowser=launchUI) def prepare(state): # Do whatever we've been asked to do... state.createWorkers() # Launch any SMTP proxies. Note that if the user hasn't specified any # SMTP proxy information in their configuration, then nothing will # happen. import smtpproxy servers, proxyPorts = smtpproxy.LoadServerInfo() proxyListeners.extend(smtpproxy.CreateProxies(servers, proxyPorts, state)) # setup info for the web interface state.buildServerStrings() def start(state): # kick everything off main(state.servers, state.proxyPorts, state.uiPort, state.launchUI) def stop(state): # Shutdown as though through the web UI. This will save the DB, allow # any open proxy connections to complete, etc. from urllib2 import urlopen from urllib import urlencode urlopen('http://localhost:%d/save' % state.uiPort, urlencode({'how': 'Save & shutdown'})).read() # =================================================================== # __main__ driver. # =================================================================== def run(): # Read the arguments. try: opts, args = getopt.getopt(sys.argv[1:], 'hbpsd:D:l:u:') except getopt.error, msg: print >>sys.stderr, str(msg) + '\n\n' + __doc__ sys.exit() runSelfTest = False for opt, arg in opts: if opt == '-h': print >>sys.stderr, __doc__ sys.exit() elif opt == '-b': state.launchUI = True elif opt == '-d': # dbm file state.useDB = True options["pop3proxy", "persistent_storage_file"] = arg elif opt == '-D': # pickle file state.useDB = False options["pop3proxy", "persistent_storage_file"] = arg elif opt == '-p': # dead option print >>sys.stderr, "-p option is no longer supported, use -D\n" print >>sys.stderr, __doc__ sys.exit() elif opt == '-l': state.proxyPorts = [_addressAndPort(arg)] elif opt == '-u': state.uiPort = int(arg) # Let the user know what they are using... print get_version_string("POP3 Proxy") print "and engine %s.\n" % (get_version_string(),) prepare(state=state) if 0 <= len(args) <= 2: # Normal usage, with optional server name and port number. if len(args) == 1: state.servers = [(args[0], 110)] elif len(args) == 2: state.servers = [(args[0], int(args[1]))] # Default to listening on port 110 for command-line-specified servers. if len(args) > 0 and state.proxyPorts == []: state.proxyPorts = [('', 110)] start(state=state) else: print >>sys.stderr, __doc__ if __name__ == '__main__': run() --- NEW FILE: sb-smtpproxy.py --- #!/usr/bin/env python """A SMTP proxy to train a Spambayes database. You point SMTP Proxy at your SMTP server(s) and configure your email client(s) to send mail through the proxy (i.e. usually this means you use localhost as the outgoing server). To setup, enter appropriate values in your Spambayes configuration file in the "SMTP Proxy" section (in particular: "remote_servers", "listen_ports", and "use_cached_message"). This configuration can also be carried out via the web user interface offered by POP3 Proxy and IMAP Filter. To use, simply forward/bounce mail that you wish to train to the appropriate address (defaults to spambayes_spam@localhost and spambayes_ham@localhost). All other mail is sent normally. (Note that IMAP Filter and POP3 Proxy users should not execute this script; launching of SMTP Proxy will be taken care of by those applicatons). There are two main forms of operation. With both, mail to two (user-configurable) email addresses is intercepted by the proxy (and is *not* sent to the SMTP server) and used as training data for a Spambayes database. All other mail is simply relayed to the SMTP server. If the "use_cached_message" option is False, the proxy uses the message sent as training data. This option is suitable for those not using POP3 Proxy or IMAP Filter, or for those that are confident that their mailer will forward/bounce messages in an unaltered form. If the "use_cached_message" option is True, the proxy examines the message for a unique spambayes identification number. It then tries to find this message in the pop3proxy caches and on the imap servers. It then retrieves the message from the cache/server and uses *this* as the training data. This method is suitable for those using POP3 Proxy and/or IMAP Filter, and avoids any potential problems with the mailer altering messages before forwarding/bouncing them. Usage: smtpproxy [options] note: option values with spaces must be enclosed in double quotes options: -d dbname : pickled training database filename -D dbname : dbm training database filename -h : help -v : verbose mode """ # This module is part of the spambayes project, which is Copyright 2002-3 # The Python Software Foundation and is covered by the Python Software # Foundation license. __author__ = "Tony Meyer " __credits__ = "Tim Stone, all the Spambayes folk." try: True, False except NameError: # Maintain compatibility with Python 2.2 True, False = 1, 0 todo = """ o It would be nice if spam/ham could be bulk forwarded to the proxy, rather than one by one. This would require separating the different messages and extracting the correct ids. Simply changing to find *all* the ids in a message, rather than stopping after one *might* work, but I don't really know. Richie Hindle suggested something along these lines back in September '02. o Suggestions? Testing: o Test with as many clients as possible to check that the id is correctly extracted from the forwarded/bounced message. MUA information: A '*' in the Header column signifies that the smtpproxy can extract the id from the headers only. A '*' in the Body column signifies that the smtpproxy can extract the id from the body of the message, if it is there. Header Body *** Windows 2000 MUAs *** Eudora 5.2 Forward * * Eudora 5.2 Redirect * Netscape Messenger (4.7) Forward (inline) * * Netscape Messenger (4.7) Forward (quoted) Plain * Netscape Messenger (4.7) Forward (quoted) HTML * Netscape Messenger (4.7) Forward (quoted) Plain & HTML * Netscape Messenger (4.7) Forward (attachment) Plain * * Netscape Messenger (4.7) Forward (attachment) HTML * * Netscape Messenger (4.7) Forward (attachment) Plain & HTML * * Outlook Express 6 Forward HTML (Base64) * Outlook Express 6 Forward HTML (None) * Outlook Express 6 Forward HTML (QP) * Outlook Express 6 Forward Plain (Base64) * Outlook Express 6 Forward Plain (None) * Outlook Express 6 Forward Plain (QP) * Outlook Express 6 Forward Plain (uuencoded) * http://www.endymion.com/products/mailman Forward * M2 (Opera Mailer 7.01) Forward * M2 (Opera Mailer 7.01) Redirect * * The Bat! 1.62i Forward (RFC Headers not visible) * The Bat! 1.62i Forward (RFC Headers visible) * * The Bat! 1.62i Redirect * The Bat! 1.62i Alternative Forward * * The Bat! 1.62i Custom Template * * AllegroMail 2.5.0.2 Forward * AllegroMail 2.5.0.2 Redirect * PocoMail 2.6.3 Bounce * PocoMail 2.6.3 Bounce * Pegasus Mail 4.02 Forward (all headers option set) * * Pegasus Mail 4.02 Forward (all headers option not set) * Calypso 3 Forward * Calypso 3 Redirect * * Becky! 2.05.10 Forward * Becky! 2.05.10 Redirect * Becky! 2.05.10 Redirect as attachment * * Mozilla Mail 1.2.1 Forward (attachment) * * Mozilla Mail 1.2.1 Forward (inline, plain) *1 * Mozilla Mail 1.2.1 Forward (inline, plain & html) *1 * Mozilla Mail 1.2.1 Forward (inline, html) *1 * *1 The header method will only work if auto-include original message is set, and if view all headers is true. """ import string import re import socket import asyncore import asynchat import getopt import sys import os from spambayes import Dibbler from spambayes import storage from spambayes.message import sbheadermessage_from_string from spambayes.tokenizer import textparts from spambayes.tokenizer import try_to_repair_damaged_base64 from spambayes.Options import options from pop3proxy import _addressPortStr, ServerLineReader from pop3proxy import _addressAndPort class SMTPProxyBase(Dibbler.BrighterAsyncChat): """An async dispatcher that understands SMTP and proxies to a SMTP server, calling `self.onTransaction(command, args)` for each transaction. self.onTransaction() should return the command to pass to the proxied server - the command can be the verbatim command or a processed version of it. The special command 'KILL' kills it (passing a 'QUIT' command to the server). """ def __init__(self, clientSocket, serverName, serverPort): Dibbler.BrighterAsyncChat.__init__(self, clientSocket) self.request = '' self.set_terminator('\r\n') self.command = '' # The SMTP command being processed... self.args = '' # ...and its arguments self.isClosing = False # Has the server closed the socket? self.inData = False self.data = "" self.blockData = False self.serverSocket = ServerLineReader(serverName, serverPort, self.onServerLine) def onTransaction(self, command, args): """Overide this. Takes the raw command and returns the (possibly processed) command to pass to the email client.""" raise NotImplementedError def onProcessData(self, data): """Overide this. Takes the raw data and returns the (possibly processed) data to pass back to the email client.""" raise NotImplementedError def onServerLine(self, line): """A line of response has been received from the SMTP server.""" # Has the server closed its end of the socket? if not line: self.isClosing = True # We don't process the return, just echo the response. self.push(line) self.onResponse() def collect_incoming_data(self, data): """Asynchat override.""" self.request = self.request + data def found_terminator(self): """Asynchat override.""" verb = self.request.strip().upper() if verb == 'KILL': self.socket.shutdown(2) self.close() raise SystemExit if self.request.strip() == '': # Someone just hit the Enter key. self.command = self.args = '' else: # A proper command. if self.request[:10].upper() == "MAIL FROM:": splitCommand = self.request.split(":", 1) elif self.request[:8].upper() == "RCPT TO:": splitCommand = self.request.split(":", 1) else: splitCommand = self.request.strip().split(None, 1) self.command = splitCommand[0] self.args = splitCommand[1:] if self.inData == True: self.data += self.request + '\r\n' if self.request == ".": self.inData = False cooked = self.onProcessData(self.data) self.data = "" if self.blockData == False: self.serverSocket.push(cooked) else: self.push("250 OK\r\n") else: cooked = self.onTransaction(self.command, self.args) if cooked is not None: self.serverSocket.push(cooked + '\r\n') self.command = self.args = self.request = '' def onResponse(self): # If onServerLine() decided that the server has closed its # socket, close this one when the response has been sent. if self.isClosing: self.close_when_done() # Reset. self.command = '' self.args = '' self.isClosing = False class BayesSMTPProxyListener(Dibbler.Listener): """Listens for incoming email client connections and spins off BayesSMTPProxy objects to serve them.""" def __init__(self, serverName, serverPort, proxyPort, trainer): proxyArgs = (serverName, serverPort, trainer) Dibbler.Listener.__init__(self, proxyPort, BayesSMTPProxy, proxyArgs) print 'SMTP Listener on port %s is proxying %s:%d' % \ (_addressPortStr(proxyPort), serverName, serverPort) class BayesSMTPProxy(SMTPProxyBase): """Proxies between an email client and a SMTP server, inserting judgement headers. It acts on the following SMTP commands: o RCPT TO: o Checks if the recipient address matches the key ham or spam addresses, and if so notes this and does not forward a command to the proxied server. In all other cases simply passes on the verbatim command. o DATA: o Notes that we are in the data section. If (from the RCPT TO information) we are receiving a ham/spam message to train on, then do not forward the command on. Otherwise forward verbatim. Any other commands are merely passed on verbatim to the server. """ def __init__(self, clientSocket, serverName, serverPort, trainer): SMTPProxyBase.__init__(self, clientSocket, serverName, serverPort) self.handlers = {'RCPT TO': self.onRcptTo, 'DATA': self.onData, 'MAIL FROM': self.onMailFrom} self.trainer = trainer self.isClosed = False self.train_as_ham = False self.train_as_spam = False def send(self, data): try: return SMTPProxyBase.send(self, data) except socket.error: # The email client has closed the connection - 40tude Dialog # does this immediately after issuing a QUIT command, # without waiting for the response. self.close() def close(self): # This can be called multiple times by async. if not self.isClosed: self.isClosed = True SMTPProxyBase.close(self) def stripAddress(self, address): """ Strip the leading & trailing <> from an address. Handy for getting FROM: addresses. """ if '<' in address: start = string.index(address, '<') + 1 end = string.index(address, '>') return address[start:end] else: return address def splitTo(self, address): """Return 'address' as undressed (host, fulladdress) tuple. Handy for use with TO: addresses.""" start = string.index(address, '<') + 1 sep = string.index(address, '@') + 1 end = string.index(address, '>') return (address[sep:end], address[start:end],) def onTransaction(self, command, args): handler = self.handlers.get(command.upper(), self.onUnknown) return handler(command, args) def onProcessData(self, data): if self.train_as_spam: self.trainer.train(data, True) self.train_as_spam = False return "" elif self.train_as_ham: self.trainer.train(data, False) self.train_as_ham = False return "" return data def onRcptTo(self, command, args): toHost, toFull = self.splitTo(args[0]) if toFull == options["smtpproxy", "spam_address"]: self.train_as_spam = True self.train_as_ham = False self.blockData = True self.push("250 OK\r\n") return None elif toFull == options["smtpproxy", "ham_address"]: self.train_as_ham = True self.train_as_spam = False self.blockData = True self.push("250 OK\r\n") return None else: self.blockData = False return "%s:%s" % (command, ' '.join(args)) def onData(self, command, args): self.inData = True if self.train_as_ham == True or self.train_as_spam == True: self.push("250 OK\r\n") return None rv = command for arg in args: rv += ' ' + arg return rv def onMailFrom(self, command, args): """Just like the default handler, but has the necessary colon.""" rv = "%s:%s" % (command, ' '.join(args)) return rv def onUnknown(self, command, args): """Default handler.""" return self.request class SMTPTrainer(object): def __init__(self, classifier, state=None, imap=None): self.classifier = classifier self.state = state self.imap = imap def extractSpambayesID(self, data): msg = message_from_string(data) # The nicest MUA is one that forwards the header intact. id = msg.get(options["Headers", "mailid_header_name"]) if id is not None: return id # Some MUAs will put it in the body somewhere, while others will # put it in an attached MIME message. id = self._find_id_in_text(msg.as_string()) if id is not None: return id # the message might be encoded for part in textparts(msg): # Decode, or take it as-is if decoding fails. try: text = part.get_payload(decode=True) except: text = part.get_payload(decode=False) if text is not None: text = try_to_repair_damaged_base64(text) if text is not None: id = self._find_id_in_text(text) return id return None header_pattern = re.escape(options["Headers", "mailid_header_name"]) # A MUA might enclose the id in a table, thus the convoluted re pattern # (Mozilla Mail does this with inline html) header_pattern += r":\s*(\\s*\\s*)?([\d\-]+)" header_re = re.compile(header_pattern) def _find_id_in_text(self, text): mo = self.header_re.search(text) if mo is None: return None return mo.group(2) def train(self, msg, isSpam): try: use_cached = options["smtpproxy", "use_cached_message"] except KeyError: use_cached = True if use_cached: id = self.extractSpambayesID(msg) if id is None: print "Could not extract id" return self.train_cached_message(id, isSpam) # Otherwise, train on the forwarded/bounced message. msg = sbheadermessage_from_string(msg) id = msg.setIdFromPayload() msg.delSBHeaders() if id is None: # No id, so we don't have any reliable method of remembering # information about this message, so we just assume that it # hasn't been trained before. We could generate some sort of # checksum for the message and use that as an id (this would # mean that we didn't need to store the id with the message) # but that might be a little unreliable. self.classifier.learn(msg.asTokens(), isSpam) else: if msg.GetTrained() == (not isSpam): self.classifier.unlearn(msg.asTokens(), not isSpam) msg.RememberTrained(None) if msg.GetTrained() is None: self.classifier.learn(msg.asTokens(), isSpam) msg.RememberTrained(isSpam) def train_cached_message(self, id, isSpam): if not self.train_message_in_pop3proxy_cache(id, isSpam) and \ not self.train_message_on_imap_server(id, isSpam): print "Could not find message (%s); perhaps it was " + \ "deleted from the POP3Proxy cache or the IMAP " + \ "server. This means that no training was done." % (id, ) def train_message_in_pop3proxy_cache(self, id, isSpam): if self.state is None: return False sourceCorpus = None for corpus in [self.state.unknownCorpus, self.state.hamCorpus, self.state.spamCorpus]: if corpus.get(id) is not None: sourceCorpus = corpus break if corpus is None: return False if isSpam == True: targetCorpus = self.state.spamCorpus else: targetCorpus = self.state.hamCorpus targetCorpus.takeMessage(id, sourceCorpus) self.classifier.store() def train_message_on_imap_server(self, id, isSpam): if self.imap is None: return False msg = self.imap.FindMessage(id) if msg is None: return False if msg.GetTrained() == (not isSpam): msg.get_substance() msg.delSBHeaders() self.classifier.unlearn(msg.asTokens(), not isSpam) msg.RememberTrained(None) if msg.GetTrained() is None: msg.get_substance() msg.delSBHeaders() self.classifier.learn(msg.asTokens(), isSpam) msg.RememberTrained(isSpam) def LoadServerInfo(): # Load the proxy settings servers = [] proxyPorts = [] if options["smtpproxy", "remote_servers"]: for server in options["smtpproxy", "remote_servers"]: server = server.strip() if server.find(':') > -1: server, port = server.split(':', 1) else: port = '25' servers.append((server, int(port))) if options["smtpproxy", "listen_ports"]: splitPorts = options["smtpproxy", "listen_ports"] proxyPorts = map(_addressAndPort, splitPorts) if len(servers) != len(proxyPorts): print "smtpproxy:remote_servers & smtpproxy:listen_ports are " + \ "different lengths!" sys.exit() return servers, proxyPorts def CreateProxies(servers, proxyPorts, trainer): """Create BayesSMTPProxyListeners for all the given servers.""" # allow for old versions of pop3proxy if not isinstance(trainer, SMTPTrainer): trainer = SMTPTrainer(trainer.bayes, trainer) proxyListeners = [] for (server, serverPort), proxyPort in zip(servers, proxyPorts): listener = BayesSMTPProxyListener(server, serverPort, proxyPort, trainer) proxyListeners.append(listener) return proxyListeners def main(): """Runs the proxy until a 'KILL' command is received or someone hits Ctrl+Break.""" try: opts, args = getopt.getopt(sys.argv[1:], 'hvd:D:') except getopt.error, msg: print >>sys.stderr, str(msg) + '\n\n' + __doc__ sys.exit() bdbname = options["Storage", "persistent_storage_file"] useDBM = options["Storage", "persistent_use_database"] for opt, arg in opts: if opt == '-h': print >>sys.stderr, __doc__ sys.exit() elif opt == '-d': useDBM = False bdbname = arg elif opt == '-D': useDBM = True bdbname = arg elif opt == '-v': options["globals", "verbose"] = True bdbname = os.path.expanduser(bdbname) if options["globals", "verbose"]: print "Loading database %s..." % (bdbname), if useDBM: classifier = storage.DBDictClassifier(bdbname) else: classifier = storage.PickledClassifier(bdbname) if options["globals", "verbose"]: print "Done." servers, proxyPorts = LoadServerInfo() trainer = SMTPTrainer(classifier) proxyListeners = CreateProxies(servers, proxyPorts, trainer) Dibbler.run() if __name__ == '__main__': main() --- NEW FILE: sb-unheader.py --- #!/usr/bin/env python """ unheader.py: cleans headers from email messages. By default, this removes SpamAssassin headers, specify a pattern with -p to supply new headers to remove. This is often needed because existing spamassassin headers can provide killer spam clues, for all the wrong reasons. """ import re import sys import os import glob import mailbox import email.Parser import email.Message import email.Generator import getopt def unheader(msg, pat): pat = re.compile(pat) for hdr in msg.keys(): if pat.match(hdr): del msg[hdr] # remain compatible with 2.2.1 - steal replace_header from 2.3 source class Message(email.Message.Message): def replace_header(self, _name, _value): """Replace a header. Replace the first matching header found in the message, retaining header order and case. If no matching header was found, a KeyError is raised. """ _name = _name.lower() for i, (k, v) in zip(range(len(self._headers)), self._headers): if k.lower() == _name: self._headers[i] = (k, _value) break else: raise KeyError, _name class Parser(email.Parser.HeaderParser): def __init__(self): email.Parser.Parser.__init__(self, Message) def deSA(msg): if msg['X-Spam-Status']: if msg['X-Spam-Status'].startswith('Yes'): pct = msg['X-Spam-Prev-Content-Type'] if pct: msg['Content-Type'] = pct pcte = msg['X-Spam-Prev-Content-Transfer-Encoding'] if pcte: msg['Content-Transfer-Encoding'] = pcte subj = re.sub(r'\*\*\*\*\*SPAM\*\*\*\*\* ', '', msg['Subject'] or "") if subj != msg["Subject"]: msg.replace_header("Subject", subj) body = msg.get_payload() newbody = [] at_start = 1 for line in body.splitlines(): if at_start and line.startswith('SPAM: '): continue elif at_start: at_start = 0 newbody.append(line) msg.set_payload("\n".join(newbody)) unheader(msg, "X-Spam-") def process_message(msg, dosa, pats): if pats is not None: unheader(msg, pats) if dosa: deSA(msg) def process_mailbox(f, dosa=1, pats=None): gen = email.Generator.Generator(sys.stdout, maxheaderlen=0) for msg in mailbox.PortableUnixMailbox(f, Parser().parse): process_message(msg, dosa, pats) gen(msg, unixfrom=1) def process_maildir(d, dosa=1, pats=None): parser = Parser() for fn in glob.glob(os.path.join(d, "cur", "*")): print ("reading from %s..." % fn), file = open(fn) msg = parser.parse(file) process_message(msg, dosa, pats) tmpfn = os.path.join(d, "tmp", os.path.basename(fn)) tmpfile = open(tmpfn, "w") print "writing to %s" % tmpfn email.Generator.Generator(tmpfile, maxheaderlen=0)(msg, unixfrom=0) os.rename(tmpfn, fn) def usage(): print >> sys.stderr, "usage: unheader.py [ -p pat ... ] [ -s ] folder" print >> sys.stderr, "-p pat gives a regex pattern used to eliminate unwanted headers" print >> sys.stderr, "'-p pat' may be given multiple times" print >> sys.stderr, "-s tells not to remove SpamAssassin headers" print >> sys.stderr, "-d means treat folder as a Maildir" def main(args): headerpats = [] dosa = 1 ismbox = 1 try: opts, args = getopt.getopt(args, "p:shd") except getopt.GetoptError: usage() sys.exit(1) else: for opt, arg in opts: if opt == "-h": usage() sys.exit(0) elif opt == "-p": headerpats.append(arg) elif opt == "-s": dosa = 0 elif opt == "-d": ismbox = 0 pats = headerpats and "|".join(headerpats) or None if len(args) != 1: usage() sys.exit(1) if ismbox: f = file(args[0]) process_mailbox(f, dosa, pats) else: process_maildir(args[0], dosa, pats) if __name__ == "__main__": main(sys.argv[1:]) --- NEW FILE: sb-upload.py --- #!/usr/bin/env python """ Read a message or a mailbox file on standard input, upload it to a web browser and write it to standard output. usage: %(progname)s [-h] [-n] [-s server] [-p port] [-r N] Options: -h, --help - print help and exit -n, --null - suppress writing to standard output (default %(null)s) -s, --server= - provide alternate web server (default %(server)s) -p, --port= - provide alternate server port (default %(port)s) -r, --prob= - feed the message to the trainer w/ prob N [0.0...1.0] """ import sys import httplib import mimetypes import getopt import random from spambayes.Options import options progname = sys.argv[0] __author__ = "Skip Montanaro " __credits__ = "Spambayes gang, Wade Leftwich" try: True, False except NameError: # Maintain compatibility with Python 2.2 True, False = 1, 0 # appropriated verbatim from a recipe by Wade Leftwich in the Python # Cookbook: http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306 def post_multipart(host, selector, fields, files): """ Post fields and files to an http host as multipart/form-data. fields is a sequence of (name, value) elements for regular form fields. files is a sequence of (name, filename, value) elements for data to be uploaded as files. Return the server's response page. """ content_type, body = encode_multipart_formdata(fields, files) h = httplib.HTTP(host) h.putrequest('POST', selector) h.putheader('content-type', content_type) h.putheader('content-length', str(len(body))) h.endheaders() h.send(body) errcode, errmsg, headers = h.getreply() return h.file.read() def encode_multipart_formdata(fields, files): """ fields is a sequence of (name, value) elements for regular form fields. files is a sequence of (name, filename, value) elements for data to be uploaded as files. Return (content_type, body) ready for httplib.HTTP instance """ BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$' CRLF = '\r\n' L = [] for (key, value) in fields: L.append('--' + BOUNDARY) L.append('Content-Disposition: form-data; name="%s"' % key) L.append('') L.append(value) for (key, filename, value) in files: L.append('--' + BOUNDARY) L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename)) L.append('Content-Type: %s' % get_content_type(filename)) L.append('') L.append(value) L.append('--' + BOUNDARY + '--') L.append('') body = CRLF.join(L) content_type = 'multipart/form-data; boundary=%s' % BOUNDARY return content_type, body def get_content_type(filename): return mimetypes.guess_type(filename)[0] or 'application/octet-stream' def usage(*args): defaults = {} for d in args: defaults.update(d) print __doc__ % defaults def main(argv): null = False server = "localhost" port = options["html_ui", "port"] prob = 1.0 try: opts, args = getopt.getopt(argv, "hns:p:r:", ["help", "null", "server=", "port=", "prob="]) except getopt.error: usage(globals(), locals()) sys.exit(1) for opt, arg in opts: if opt in ("-h", "--help"): usage(globals(), locals()) sys.exit(0) elif opt in ("-n", "--null"): null = True elif opt in ("-s", "--server"): server = arg elif opt in ("-p", "--port"): port = int(arg) elif opt in ("-r", "--prob"): n = float(arg) if n < 0.0 or n > 1.0: usage(globals(), locals()) sys.exit(1) prob = n if args: usage(globals(), locals()) sys.exit(1) data = sys.stdin.read() sys.stdout.write(data) if random.random() < prob: try: post_multipart("%s:%d"%(server,port), "/upload", [], [('file', 'message.dat', data)]) except: # not an error if the server isn't responding pass if __name__ == "__main__": main(sys.argv[1:]) --- NEW FILE: sb-xmlrpcserver.py --- #! /usr/bin/env python # A server version of hammie.py """Usage: %(program)s [options] IP:PORT Where: -h show usage and exit -p FILE use file as the persistent store. loads data from this file if it exists, and saves data to this file at the end. Default: %(DEFAULTDB)s -d use the DBM store instead of cPickle. The file is larger and creating it is slower, but checking against it is much faster, especially for large word databases. IP IP address to bind (use 0.0.0.0 to listen on all IPs of this machine) PORT Port number to listen to. """ import SimpleXMLRPCServer import getopt import sys import traceback import xmlrpclib from spambayes import hammie try: True, False except NameError: # Maintain compatibility with Python 2.2 True, False = 1, 0 program = sys.argv[0] # For usage(); referenced by docstring above # Default DB path DEFAULTDB = hammie.DEFAULTDB class XMLHammie(hammie.Hammie): def score(self, msg, *extra): try: msg = msg.data except AttributeError: pass return xmlrpclib.Binary(hammie.Hammie.score(self, msg, *extra)) def filter(self, msg, *extra): try: msg = msg.data except AttributeError: pass return xmlrpclib.Binary(hammie.Hammie.filter(self, msg, *extra)) class HammieHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): def do_POST(self): """Handles the HTTP POST request. Attempts to interpret all HTTP POST requests as XML-RPC calls, which are forwarded to the _dispatch method for handling. This one also prints out tracebacks, to help me debug :) """ try: # get arguments data = self.rfile.read(int(self.headers["content-length"])) params, method = xmlrpclib.loads(data) # generate response try: response = self._dispatch(method, params) # wrap response in a singleton tuple response = (response,) except: traceback.print_exc() # report exception back to server response = xmlrpclib.dumps( xmlrpclib.Fault(1, "%s:%s" % (sys.exc_type, sys.exc_value)) ) else: response = xmlrpclib.dumps(response, methodresponse=1) except: # internal error, report as HTTP server error traceback.print_exc() print `data` self.send_response(500) self.end_headers() else: # got a valid XML RPC response self.send_response(200) self.send_header("Content-type", "text/xml") self.send_header("Content-length", str(len(response))) self.end_headers() self.wfile.write(response) # shut down the connection self.wfile.flush() self.connection.shutdown(1) def usage(code, msg=''): """Print usage message and sys.exit(code).""" if msg: print >> sys.stderr, msg print >> sys.stderr print >> sys.stderr, __doc__ % globals() sys.exit(code) def main(): """Main program; parse options and go.""" try: opts, args = getopt.getopt(sys.argv[1:], 'hdp:') except getopt.error, msg: usage(2, msg) pck = DEFAULTDB usedb = False for opt, arg in opts: if opt == '-h': usage(0) elif opt == '-p': pck = arg elif opt == "-d": usedb = True if len(args) != 1: usage(2, "IP:PORT not specified") ip, port = args[0].split(":") port = int(port) bayes = hammie.createbayes(pck, usedb) h = XMLHammie(bayes) server = SimpleXMLRPCServer.SimpleXMLRPCServer((ip, port), HammieHandler) server.register_instance(h) server.serve_forever() if __name__ == "__main__": main() --- sb-overkill.py DELETED --- From anadelonbrin at users.sourceforge.net Thu Sep 4 19:15:30 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Thu Sep 4 21:15:34 2003 Subject: [Spambayes-checkins] spambayes/contrib nway.py,1.2,1.3 Message-ID: Update of /cvsroot/spambayes/spambayes/contrib In directory sc8-pr-cvs1:/tmp/cvs-serv14102/contrib Modified Files: nway.py Log Message: Remove backwards compat code for options, and update all (I hope) the remaining code that uses it. Index: nway.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/contrib/nway.py,v retrieving revision 1.2 retrieving revision 1.3 diff -C2 -d -r1.2 -r1.3 *** nway.py 13 Aug 2003 14:39:35 -0000 1.2 --- nway.py 5 Sep 2003 01:15:28 -0000 1.3 *************** *** 62,66 **** h = hammie.open(db, True, 'r') score = h.score(msg) ! if score >= Options.options.spam_cutoff: msg["X-Spambayes-Classification"] = "%s; %.2f" % (tag, score) break --- 62,66 ---- h = hammie.open(db, True, 'r') score = h.score(msg) ! if score >= Options.options["Categorization", "spam_cutoff"]: msg["X-Spambayes-Classification"] = "%s; %.2f" % (tag, score) break From anadelonbrin at users.sourceforge.net Thu Sep 4 19:15:31 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Thu Sep 4 21:15:38 2003 Subject: [Spambayes-checkins] spambayes/spambayes/resources __init__.py, 1.1, 1.2 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes/resources In directory sc8-pr-cvs1:/tmp/cvs-serv14102/spambayes/resources Modified Files: __init__.py Log Message: Remove backwards compat code for options, and update all (I hope) the remaining code that uses it. Index: __init__.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/resources/__init__.py,v retrieving revision 1.1 retrieving revision 1.2 diff -C2 -d -r1.1 -r1.2 *** __init__.py 17 Jan 2003 20:21:13 -0000 1.1 --- __init__.py 5 Sep 2003 01:15:28 -0000 1.2 *************** *** 1,2 **** ! # See scanning__init__.py for how to change these resources. --- 1,36 ---- + """Design-time __init__.py for resourcepackage ! This is the scanning version of __init__.py for your ! resource modules. You replace it with a blank or doc-only ! init when ready to release. ! """ ! try: ! __file__ ! except NameError: ! pass ! else: ! import os ! if os.path.splitext(os.path.basename( __file__ ))[0] == "__init__": ! try: ! from resourcepackage import package, defaultgenerators ! generators = defaultgenerators.generators.copy() ! ! ### CUSTOMISATION POINT ! ## import specialised generators here, such as for wxPython ! #from resourcepackage import wxgenerators ! #generators.update( wxgenerators.generators ) ! except ImportError: ! pass ! else: ! package = package.Package( ! packageName = __name__, ! directory = os.path.dirname( os.path.abspath(__file__) ), ! generators = generators, ! ) ! package.scan( ! ### CUSTOMISATION POINT ! ## force true -> always re-loads from external files, otherwise ! ## only reloads if the file is newer than the generated .py file. ! # force = 1, ! ) ! From anadelonbrin at users.sourceforge.net Thu Sep 4 19:15:31 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Thu Sep 4 21:15:42 2003 Subject: [Spambayes-checkins] spambayes/testtools incremental.py, 1.4, 1.5 simplexloop.py, 1.3, 1.4 timcv.py, 1.3, 1.4 timtest.py, 1.4, 1.5 weaktest.py, 1.3, 1.4 Message-ID: Update of /cvsroot/spambayes/spambayes/testtools In directory sc8-pr-cvs1:/tmp/cvs-serv14102/testtools Modified Files: incremental.py simplexloop.py timcv.py timtest.py weaktest.py Log Message: Remove backwards compat code for options, and update all (I hope) the remaining code that uses it. Index: incremental.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/testtools/incremental.py,v retrieving revision 1.4 retrieving revision 1.5 diff -C2 -d -r1.4 -r1.5 *** incremental.py 28 Feb 2003 17:57:53 -0000 1.4 --- incremental.py 5 Sep 2003 01:15:29 -0000 1.5 *************** *** 111,116 **** for example in stream: prob = guess(example) ! is_ham_guessed = prob < options.ham_cutoff ! is_spam_guessed = prob >= options.spam_cutoff if is_spam: self.nspam_tested += 1 --- 111,116 ---- for example in stream: prob = guess(example) ! is_ham_guessed = prob < options["Categorization", "ham_cutoff"] ! is_spam_guessed = prob >= options["Categorization", "spam_cutoff"] if is_spam: self.nspam_tested += 1 *************** *** 176,180 **** >>> from spambayes.classifier import Bayes >>> from spambayes.Options import options ! >>> options.ham_cutoff = options.spam_cutoff = 0.5 >>> good1 = _Example('', ['a', 'b', 'c']) --- 176,180 ---- >>> from spambayes.classifier import Bayes >>> from spambayes.Options import options ! >>> options["Categorization", "ham_cutoff"] = options["Categorization", "spam_cutoff"] = 0.5 >>> good1 = _Example('', ['a', 'b', 'c']) Index: simplexloop.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/testtools/simplexloop.py,v retrieving revision 1.3 retrieving revision 1.4 diff -C2 -d -r1.3 -r1.4 *** simplexloop.py 29 Jan 2003 03:23:35 -0000 1.3 --- simplexloop.py 5 Sep 2003 01:15:29 -0000 1.4 *************** *** 33,39 **** from spambayes import Options ! start = (Options.options.unknown_word_prob, ! Options.options.minimum_prob_strength, ! Options.options.unknown_word_strength) err = (0.01, 0.01, 0.01) --- 33,39 ---- from spambayes import Options ! start = (Options.options["Tokenizer", "unknown_word_prob", ! Options.options["Tokenzier", "minimum_prob_strength"], ! Options.options["Tokenizer", "unknown_word_strength"]) err = (0.01, 0.01, 0.01) Index: timcv.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/testtools/timcv.py,v retrieving revision 1.3 retrieving revision 1.4 diff -C2 -d -r1.3 -r1.4 *** timcv.py 8 Mar 2003 00:33:23 -0000 1.3 --- timcv.py 5 Sep 2003 01:15:29 -0000 1.4 *************** *** 73,78 **** print options.display() ! hamdirs = [options.ham_directories % i for i in range(1, nsets+1)] ! spamdirs = [options.spam_directories % i for i in range(1, nsets+1)] d = TestDriver.Driver() --- 73,80 ---- print options.display() ! hamdirs = [options["TestDriver", "ham_directories"] % \ ! i for i in range(1, nsets+1)] ! spamdirs = [options["TestDriver", "spam_directories"] % \ ! i for i in range(1, nsets+1)] d = TestDriver.Driver() *************** *** 91,95 **** if i > 0: ! if options.build_each_classifier_from_scratch: # Build a new classifier from the other sets. d.new_classifier() --- 93,97 ---- if i > 0: ! if options["CV Driver", "build_each_classifier_from_scratch"]: # Build a new classifier from the other sets. d.new_classifier() *************** *** 114,118 **** d.finishtest() ! if i < nsets - 1 and not options.build_each_classifier_from_scratch: # Add this set back in. d.train(hamstream, spamstream) --- 116,121 ---- d.finishtest() ! if i < nsets - 1 and not options["CV Driver", ! "build_each_classifier_from_scratch"]: # Add this set back in. d.train(hamstream, spamstream) Index: timtest.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/testtools/timtest.py,v retrieving revision 1.4 retrieving revision 1.5 diff -C2 -d -r1.4 -r1.5 *** timtest.py 30 Apr 2003 06:43:28 -0000 1.4 --- timtest.py 5 Sep 2003 01:15:29 -0000 1.5 *************** *** 58,63 **** print options.display() ! spamdirs = [options.spam_directories % i for i in range(1, nsets+1)] ! hamdirs = [options.ham_directories % i for i in range(1, nsets+1)] spamhamdirs = zip(spamdirs, hamdirs) --- 58,65 ---- print options.display() ! spamdirs = [options["TestDriver", "spam_directories"] % \ ! i for i in range(1, nsets+1)] ! hamdirs = [options["TestDriver", "ham_directories"] % \ ! i for i in range(1, nsets+1)] spamhamdirs = zip(spamdirs, hamdirs) Index: weaktest.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/testtools/weaktest.py,v retrieving revision 1.3 retrieving revision 1.4 diff -C2 -d -r1.3 -r1.4 *** weaktest.py 29 Jan 2003 03:23:35 -0000 1.3 --- weaktest.py 5 Sep 2003 01:15:29 -0000 1.4 *************** *** 62,79 **** class UnsureAndFalses(TrainDecision): def spamtrain(self,scr): ! if scr < options.spam_cutoff: return TRAIN_AS_SPAM def hamtrain(self,scr): ! if scr > options.ham_cutoff: return TRAIN_AS_HAM class UnsureOnly(TrainDecision): def spamtrain(self,scr): ! if options.ham_cutoff < scr < options.spam_cutoff: return TRAIN_AS_SPAM def hamtrain(self,scr): ! if options.ham_cutoff < scr < options.spam_cutoff: return TRAIN_AS_HAM --- 62,81 ---- class UnsureAndFalses(TrainDecision): def spamtrain(self,scr): ! if scr < options["Categorization", "spam_cutoff"]: return TRAIN_AS_SPAM def hamtrain(self,scr): ! if scr > options["Categorization", "ham_cutoff"]: return TRAIN_AS_HAM class UnsureOnly(TrainDecision): def spamtrain(self,scr): ! if options["Categorization", "ham_cutoff"] < scr < \ ! options["Categorization", "spam_cutoff"]: return TRAIN_AS_SPAM def hamtrain(self,scr): ! if options["Categorization", "ham_cutoff"] < scr < \ ! options["Categorization", "spam_cutoff"]: return TRAIN_AS_HAM *************** *** 96,102 **** class OwnDecision(TrainDecision): def hamtrain(self,scr): ! if scr < options.ham_cutoff: return TRAIN_AS_HAM ! elif scr > options.spam_cutoff: return TRAIN_AS_SPAM --- 98,104 ---- class OwnDecision(TrainDecision): def hamtrain(self,scr): ! if scr < options["Categorization", "ham_cutoff"]: return TRAIN_AS_HAM ! elif scr > options["Categorization", "spam_cutoff"]: return TRAIN_AS_SPAM *************** *** 146,151 **** print options.display() ! spamdirs = [options.spam_directories % i for i in range(1, nsets+1)] ! hamdirs = [options.ham_directories % i for i in range(1, nsets+1)] spamfns = [(x,y,1) for x in spamdirs for y in os.listdir(x)] --- 148,155 ---- print options.display() ! spamdirs = [options["TestDriver", "spam_directories"] % \ ! i for i in range(1, nsets+1)] ! hamdirs = [options["TestDriver", "ham_directories"] % \ ! i for i in range(1, nsets+1)] spamfns = [(x,y,1) for x in spamdirs for y in os.listdir(x)] From anadelonbrin at users.sourceforge.net Thu Sep 4 19:15:31 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Thu Sep 4 21:15:46 2003 Subject: [Spambayes-checkins] spambayes/windows autoconfigure.py,1.3,1.4 Message-ID: Update of /cvsroot/spambayes/spambayes/windows In directory sc8-pr-cvs1:/tmp/cvs-serv14102/windows Modified Files: autoconfigure.py Log Message: Remove backwards compat code for options, and update all (I hope) the remaining code that uses it. Index: autoconfigure.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/windows/autoconfigure.py,v retrieving revision 1.3 retrieving revision 1.4 diff -C2 -d -r1.3 -r1.4 *** autoconfigure.py 2 Sep 2003 08:29:01 -0000 1.3 --- autoconfigure.py 5 Sep 2003 01:15:29 -0000 1.4 *************** *** 55,63 **** --- 55,70 ---- import re import os + import sys import types import socket + import shutil import StringIO import ConfigParser + # Allow for those without SpamBayes on the PYTHONPATH + sys.path.insert(-1, os.getcwd()) + sys.path.insert(-1, os.path.dirname(os.getcwd())) + + from spambayes import OptionsClass from spambayes.Options import options, optionsPathname *************** *** 75,79 **** s.close() except socket.error: ! return port # Let's be safe and use high ports, starting at 1110 and 1025, and going up --- 82,91 ---- s.close() except socket.error: ! portStr = str(port) ! if options["pop3proxy", "listen_ports"].find(portStr) != -1 or \ ! options["smtpproxy", "listen_ports"].find(portStr) != -1: ! continue ! else: ! return port # Let's be safe and use high ports, starting at 1110 and 1025, and going up *************** *** 470,475 **** --- 482,660 ---- # option, obviously) + def configure_pegasus_mail(config_location): + """Configure Pegasus Mail to use the SpamBayes POP3 and SMTP proxies, + and configure SpamBayes to proxy the servers that Pegasus Mail was + connecting to.""" + + # We can't use ConfigParser here, as we want 'surgical' editing, + # so we want to use out OptionsClass. There is the additional trouble + # that the Pegasus Mail config file doesn't have a section header. + + pop_proxy = pop_proxy_port + smtp_proxy = smtp_proxy_port + + for filename in os.listdir(config_location): + if filename.lower().startswith("pop") or filename.lower().startswith("smt"): + full_filename = os.path.join(config_location, filename) + working_filename = "%s.tmp" % (filename, ) + shutil.copyfile(filename, working_filename) + c = OptionsClass.OptionsClass() + c.merge_file(working_filename) + server = "%s:%s" % (c.get("all", "host"), c.get("all", "port")) + if filename[:3] == "pop": + pop_proxy = move_to_next_free_port(pop_proxy) + proxy = pop_proxy + sect = "pop3proxy" + else: + smtp_proxy = move_to_next_free_port(smtp_proxy) + proxy = smtp_proxy + sect = "smtpproxy" + options[sect, "remote_servers"] += (server,) + options[sect, "listen_ports"] += (proxy,) + # Write in the new options!! + c.set("all", "host", "127.0.0.1") + c.set("all", "port", proxy) + c.update_file(working_filename) + if options["globals", "verbose"]: + print "[%s] Proxy %s on localhost:%s" % \ + (c.get("all", "title"), server, proxy) + elif filename.lower() == "IMAP.PM": + # Setup imapfilter instead. + pass + + # Pegasus Mail has a 'weight' system for determining junk mail. + # The best plan would probably be to just add to this. Something like: + rules_filename = os.path.join(config_location, "spambust.dat") + header_name = options["Headers", "classification_header_name"] + spam_tag = options["Headers", "header_spam_string"] + unsure_tag = options["Headers", "header_unsure_string"] + ham_tag = options["Headers", "header_ham_string"] + spam_weight = 500 + ham_weight = -500 + unsure_weight = -50 # leave judgement up to the rest of the rules + rule = '# SpamBayes adjustments\n' \ + 'if header "%s" contains "%s" weight %s\n' \ + 'if header "%s" contains "%s" weight %s\n' \ + 'if header "%s" contains "%s" wieght %s\n\n' % \ + (header_name, spam_tag, spam_weight, + header_name, unsure_tag, unsure_weight, + header_name, ham_tag, ham_weight) + rules_file = file(rules_filename, "a") + rules_file.write(rule) + rules_file.close() + + def configure_pocomail(): + import win32api + import win32con + + key = "Software\\Poco Systems Inc" + + pop_proxy = pop_proxy_port + smtp_proxy = smtp_proxy_port + + reg = win32api.RegOpenKeyEx(win32con.HKEY_CURRENT_USER, key) + subkey_name = "%s\\%s" % (key, win32api.RegEnumKey(reg, 0)) + reg = win32api.RegOpenKeyEx(win32con.HKEY_CURRENT_USER, + subkey_name) + pocomail_path = win32api.RegQueryValueEx(reg, "Path")[0] + + pocomail_accounts_file = os.path.join(pocomail_path, "accounts.ini") + + if os.path.exists(pocomail_accounts_file): + f = open(pocomail_accounts_file, "r") + + accountName = "" + pocomail_accounts = { } + + # Builds the dictionary with all the existing accounts. + for line in f.readlines(): + line = line.rstrip('\n\r') + if line == '': + continue + + if line[0] == '[' and line[-1] == ']': + accountName = line[1:-1] + pocomail_accounts[accountName] = { } + else: + separator = line.find('=') + optionName = line[:separator] + optionValue = line[separator + 1:] + + if optionName == "POPServer": + pop3 = optionValue.split(':') + if len(pop3) == 1: + pop3.append(110) + server = "%s:%s" % tuple(pop3) + + proxy = pop_proxy + pop_proxy = move_to_next_free_port(pop_proxy) + + if not server in options["pop3proxy", "remote_servers"]: + options["pop3proxy", "remote_servers"] += (server,) + options["pop3proxy", "listen_ports"] += (proxy,) + else: + serverIndex = 0 + for remoteServer in options["pop3proxy", + "remote_servers"]: + if remoteServer == server: + break + serverIndex += 1 + proxy = options["pop3proxy", "listen_ports"][serverIndex] + + optionValue = "%s:%s" % ('localhost', proxy) + + pocomail_accounts[accountName][optionName] = optionValue + + f.close() + f = open(pocomail_accounts_file, "w") + for accountName in pocomail_accounts.keys(): + f.write('[' + accountName + ']\n') + for optionName, optionValue in pocomail_accounts[accountName].items(): + f.write("%s=%s\n" % (optionName, optionValue)) + f.write('\n') + f.close() + + options.update_file(optionsPathname) + + # Add a filter to pocomail + pocomail_filters_file = os.path.join(pocomail_path, "filters.ini") + + if os.path.exists(pocomail_filters_file): + f = open(pocomail_filters_file, "r") + + pocomail_filters = { } + filterName = "" + + for line in f.readlines(): + line = line.rstrip('\n\r') + if line == '': continue + + if line[0] == '[' and line[-1] == ']': + filterName = line[1:-1] + pocomail_filters[filterName] = [] + elif line[0] != '{': + pocomail_filters[filterName].append(line) + f.close() + + spamBayesFilter = 'spam,X-Spambayes-Classification,move,' \ + '"Junk Mail",0,0,,,0,,,move,In,0,0,,0,,,' \ + 'move,In,0,0,,0,,,move,In,0,0,,0,,,move,' \ + 'In,0,0,,0,,,move,In,0,0,1,0' + if pocomail_filters.has_key("Incoming") and \ + spamBayesFilter not in pocomail_filters["Incoming"]: + pocomail_filters["Incoming"].append(spamBayesFilter) + + f = open(pocomail_filters_file, "w") + f.write('{ Filter list generated by PocoMail 3.01 (1661)' \ + '- Licensed Version}\n') + for filterName in pocomail_filters.keys(): + f.write('\n[' + filterName + ']\n') + for filter in pocomail_filters[filterName]: + f.write(filter + '\n') + f.close() + if __name__ == "__main__": + pmail_ini_dir = "C:\\Program Files\\PMAIL\\MAIL\\ADMIN" # XXX This is my OE key = "S-1-5-21-95318837-410984162-318601546-13224" # XXX but I presume it's different for everyone? I'll have to check on *************** *** 478,481 **** #configure_mozilla(mozilla_ini_dir) #configure_m2(m2_ini_dir) ! configure_outlook_express() pass --- 663,668 ---- #configure_mozilla(mozilla_ini_dir) #configure_m2(m2_ini_dir) ! #configure_outlook_express() ! #configure_pocomail() ! configure_pegasus_mail(pmail_ini_dir) pass From anadelonbrin at users.sourceforge.net Thu Sep 4 19:15:31 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Thu Sep 4 21:15:50 2003 Subject: [Spambayes-checkins] spambayes/spambayes Options.py, 1.69, 1.70 OptionsClass.py, 1.11, 1.12 TestDriver.py, 1.3, 1.4 classifier.py, 1.8, 1.9 dbmstorage.py, 1.7, 1.8 hammie.py, 1.7, 1.8 storage.py, 1.27, 1.28 tokenizer.py, 1.13, 1.14 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv14102/spambayes Modified Files: Options.py OptionsClass.py TestDriver.py classifier.py dbmstorage.py hammie.py storage.py tokenizer.py Log Message: Remove backwards compat code for options, and update all (I hope) the remaining code that uses it. Index: Options.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v retrieving revision 1.69 retrieving revision 1.70 diff -C2 -d -r1.69 -r1.70 *** Options.py 1 Sep 2003 06:07:27 -0000 1.69 --- Options.py 5 Sep 2003 01:15:28 -0000 1.70 *************** *** 15,21 **** To Do: ! o Get rid of the really ugly backwards compatability code (that adds ! many, many attributes to the options object) as soon as all the ! modules are changed over. """ --- 15,19 ---- To Do: ! o Suggestions? """ *************** *** 24,78 **** __all__ = ['options'] - # Backwards compatibility stuff - this will be removed at some point - # This table allows easy conversion from the (old_section, old_option) - # names to the (new_section, new_option) names. - conversion_table = {("Hammie", "clue_mailheader_cutoff"): - ("Headers", "clue_mailheader_cutoff"), - ("Hammie", "header_score_digits"): - ("Headers", "header_score_digits"), - ("Hammie", "header_score_logarithm"): - ("Headers", "header_score_logarithm"), - ("Hammie", "header_name"): - ("Headers", "classification_header_name"), - ("Hammie", "header_ham_string"): - ("Headers", "header_ham_string"), - ("Hammie", "header_spam_string"): - ("Headers", "header_spam_string"), - ("Hammie", "header_unsure_string"): - ("Headers", "header_unsure_string"), - ("Hammie", "trained_header"): - ("Headers", "trained_header_name"), - ("pop3proxy", "evidence_header_name"): - ("Headers", "evidence_header_name"), - ("pop3proxy", "mailid_header_name"): - ("Headers", "mailid_header_name"), - ("pop3proxy", "prob_header_name"): - ("Headers", "score_header_name"), - ("pop3proxy", "thermostat_header_name"): - ("Headers", "thermostat_header_name"), - ("pop3proxy", "include_evidence"): - ("Headers", "include_evidence"), - ("pop3proxy", "include_prob"): - ("Headers", "include_score"), - ("pop3proxy", "include_thermostat"): - ("Headers", "include_thermostat"), - ("pop3proxy", "ports"): - ("pop3proxy", "listen_ports"), - ("pop3proxy", "servers"): - ("pop3proxy", "remote_servers"), - ("smtpproxy", "ports"): - ("smtpproxy", "listen_ports"), - ("smtpproxy", "servers"): - ("smtpproxy", "remote_servers"), - ("hammiefilter", "persistent_storage_file"): - ("Storage", "persistent_storage_file"), - ("hammiefilter", "persistent_use_database"): - ("Storage", "persistent_use_database"), - ("pop3proxy", "persistent_storage_file"): - ("Storage", "persistent_storage_file"), - ("pop3proxy", "persistent_use_database"): - ("Storage", "persistent_use_database"), - } - # Grab the stuff from the core options class. from OptionsClass import * --- 22,25 ---- *************** *** 983,987 **** options = OptionsClass() - options.conversion_table = conversion_table # Our b/w compat hacks options.load_defaults(defaults) --- 930,933 ---- Index: OptionsClass.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/OptionsClass.py,v retrieving revision 1.11 retrieving revision 1.12 diff -C2 -d -r1.11 -r1.12 *** OptionsClass.py 25 Aug 2003 09:00:36 -0000 1.11 --- OptionsClass.py 5 Sep 2003 01:15:28 -0000 1.12 *************** *** 23,31 **** To Do: ! o Get rid of the really ugly backwards compatability code (that adds ! many, many attributes to the options object) as soon as all the ! modules are changed over. ! o Once the above is done, and we have waited a suitable time, stop ! allowing invalid options in configuration files o Find a regex expert to come up with *good* patterns for domains, email addresses, and so forth. --- 23,27 ---- To Do: ! o Stop allowing invalid options in configuration files o Find a regex expert to come up with *good* patterns for domains, email addresses, and so forth. *************** *** 491,511 **** o = klass(*args) self._options[section, o.name] = o - # A (really ugly) bit of backwards compatability - # *** This will vanish soon, so do not make use of it in - # new code *** - self._oldset(section, o.name, o.value) - def _oldset(self, section, option, value): - # A (really ugly) bit of backwards compatability - # *** This will vanish soon, so do not make use of it in - # new code *** - for (oldsect, oldopt), (newsect, newopt) in self.conversion_table.items(): - if (newsect, newopt) == (section, option): - section = oldsect - option = oldopt - setattr(self, option, value) - old_name = section[0:1].lower() + section[1:] + "_" + option - setattr(self, old_name, value) - def merge_files(self, file_list): for file in file_list: --- 487,491 ---- *************** *** 521,537 **** section = sect option = opt - # backward compatibility guff, but only if needed. - if self.conversion_table: - if opt[:len(sect) + 1].lower() == sect.lower() + '_': - opt = opt[len(sect)+1:] - if self.conversion_table.has_key((sect, opt)): - section, option = self.conversion_table[sect, opt] - else: - section = sect - option = opt - else: # no b/w compat needed. - section = sect - option = opt - # end of backward compatibility guff if not self._options.has_key((section, option)): print "Invalid option %s in section %s in file %s" % \ --- 501,504 ---- *************** *** 542,549 **** value = self.convert(section, option, value) self.set(section, option, value) - # backward compatibility guff - if self.conversion_table: - self._oldset(sect, opt, value) - # end of backward compatibility guff # not strictly necessary, but convenient shortcuts to self._options --- 509,512 ---- *************** *** 603,608 **** if self.is_valid(sect, opt, val): self._options[sect, opt].set(val) - # backwards compatibility stuff - self._oldset(sect, opt, val) else: print "Attempted to set [%s] %s with invalid value %s (%s)" % \ --- 566,569 ---- Index: TestDriver.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/TestDriver.py,v retrieving revision 1.3 retrieving revision 1.4 diff -C2 -d -r1.3 -r1.4 *** TestDriver.py 29 Jan 2003 03:23:34 -0000 1.3 --- TestDriver.py 5 Sep 2003 01:15:28 -0000 1.4 *************** *** 46,50 **** ! def printhist(tag, ham, spam, nbuckets=options.nbuckets): print print "-> Ham scores for", tag, --- 46,50 ---- ! def printhist(tag, ham, spam, nbuckets=options["TestDriver", "nbuckets"]): print print "-> Ham scores for", tag, *************** *** 55,59 **** spam.display(nbuckets) ! if not options.compute_best_cutoffs_from_histograms: return if ham.n == 0 or spam.n == 0: --- 55,59 ---- spam.display(nbuckets) ! if not options["TestDriver", "compute_best_cutoffs_from_histograms"]: return if ham.n == 0 or spam.n == 0: *************** *** 70,76 **** assert ham.nbuckets == spam.nbuckets n = ham.nbuckets ! FPW = options.best_cutoff_fp_weight ! FNW = options.best_cutoff_fn_weight ! UNW = options.best_cutoff_unsure_weight # Get running totals: {h,s}total[i] is # of ham/spam below bucket i --- 70,76 ---- assert ham.nbuckets == spam.nbuckets n = ham.nbuckets ! FPW = options["TestDriver", "best_cutoff_fp_weight"] ! FNW = options["TestDriver", "best_cutoff_fn_weight"] ! UNW = options["TestDriver", "best_cutoff_unsure_weight"] # Get running totals: {h,s}total[i] is # of ham/spam below bucket i *************** *** 137,142 **** print guts = str(msg) ! if options.show_charlimit > 0: ! guts = guts[:options.show_charlimit] print guts --- 137,142 ---- print guts = str(msg) ! if options["TestDriver", "show_charlimit"] > 0: ! guts = guts[:options["TestDriver", "show_charlimit"]] print guts *************** *** 181,185 **** def finishtest(self): ! if options.show_histograms: printhist("all in this training set:", self.trained_ham_hist, self.trained_spam_hist) --- 181,185 ---- def finishtest(self): ! if options["TestDriver", "show_histograms"]: printhist("all in this training set:", self.trained_ham_hist, self.trained_spam_hist) *************** *** 190,195 **** self.ntimes_finishtest_called += 1 ! if options.save_trained_pickles: ! fname = "%s%d.pik" % (options.pickle_basename, self.ntimes_finishtest_called) print " saving pickle to", fname --- 190,195 ---- self.ntimes_finishtest_called += 1 ! if options["TestDriver", "save_trained_pickles"]: ! fname = "%s%d.pik" % (options["TestDriver", "pickle_basename"], self.ntimes_finishtest_called) print " saving pickle to", fname *************** *** 199,209 **** def alldone(self): ! if options.show_histograms: besthamcut,bestspamcut = printhist("all runs:", self.global_ham_hist, self.global_spam_hist) else: ! besthamcut = options.ham_cutoff ! bestspamcut = options.spam_cutoff nham = self.global_ham_hist.n nspam = self.global_spam_hist.n --- 199,209 ---- def alldone(self): ! if options["TestDriver", "show_histograms"]: besthamcut,bestspamcut = printhist("all runs:", self.global_ham_hist, self.global_spam_hist) else: ! besthamcut = options["Categorization", "ham_cutoff"] ! bestspamcut = options["Categorization", "spam_cutoff"] nham = self.global_ham_hist.n nspam = self.global_spam_hist.n *************** *** 218,233 **** print "-> all runs unsure %:", (nun * 1e2 / (nham + nspam)) print "-> all runs cost: $%.2f" % ( ! nfp * options.best_cutoff_fp_weight + ! nfn * options.best_cutoff_fn_weight + ! nun * options.best_cutoff_unsure_weight) # Set back the options for the delayed calculations in self.cc ! options.ham_cutoff = besthamcut ! options.spam_cutoff = bestspamcut print self.cc ! if options.save_histogram_pickles: for f, h in (('ham', self.global_ham_hist), ('spam', self.global_spam_hist)): ! fname = "%s_%shist.pik" % (options.pickle_basename, f) print " saving %s histogram pickle to %s" %(f, fname) fp = file(fname, 'wb') --- 218,234 ---- print "-> all runs unsure %:", (nun * 1e2 / (nham + nspam)) print "-> all runs cost: $%.2f" % ( ! nfp * options["TestDriver", "best_cutoff_fp_weight"] + ! nfn * options["TestDriver", "best_cutoff_fn_weight"] + ! nun * options["TestDriver", "best_cutoff_unsure_weight"]) # Set back the options for the delayed calculations in self.cc ! options["Categorization", "ham_cutoff"] = besthamcut ! options["Categorization", "spam_cutoff"] = bestspamcut print self.cc ! if options["TestDriver", "save_histogram_pickles"]: for f, h in (('ham', self.global_ham_hist), ('spam', self.global_spam_hist)): ! fname = "%s_%shist.pik" % (options["TestDriver", ! "pickle_basename"], f) print " saving %s histogram pickle to %s" %(f, fname) fp = file(fname, 'wb') *************** *** 241,246 **** local_spam_hist = Hist() ! def new_ham(msg, prob, lo=options.show_ham_lo, ! hi=options.show_ham_hi): local_ham_hist.add(prob * 100.0) self.cc.ham(prob) --- 242,247 ---- local_spam_hist = Hist() ! def new_ham(msg, prob, lo=options["TestDriver", "show_ham_lo"], ! hi=options["TestDriver", "show_ham_hi"]): local_ham_hist.add(prob * 100.0) self.cc.ham(prob) *************** *** 251,256 **** printmsg(msg, prob, clues) ! def new_spam(msg, prob, lo=options.show_spam_lo, ! hi=options.show_spam_hi): local_spam_hist.add(prob * 100.0) self.cc.spam(prob) --- 252,257 ---- printmsg(msg, prob, clues) ! def new_spam(msg, prob, lo=options["TestDriver", "show_spam_lo"], ! hi=options["TestDriver", "show_spam_hi"]): local_spam_hist.add(prob * 100.0) self.cc.spam(prob) *************** *** 272,279 **** print "-> unsure %:", t.unsure_rate() print "-> cost: $%.2f" % ( ! t.nham_wrong * options.best_cutoff_fp_weight + ! t.nspam_wrong * options.best_cutoff_fn_weight + (t.nham_unsure + t.nspam_unsure) * ! options.best_cutoff_unsure_weight) newfpos = Set(t.false_positives()) - self.falsepos --- 273,280 ---- print "-> unsure %:", t.unsure_rate() print "-> cost: $%.2f" % ( ! t.nham_wrong * options["TestDriver", "best_cutoff_fp_weight"] + ! t.nspam_wrong * options["TestDriver", "best_cutoff_fn_weight"] + (t.nham_unsure + t.nspam_unsure) * ! options["TestDriver", "best_cutoff_unsure_weight"]) newfpos = Set(t.false_positives()) - self.falsepos *************** *** 282,286 **** if newfpos: print " new fp:", [e.tag for e in newfpos] ! if not options.show_false_positives: newfpos = () for e in newfpos: --- 283,287 ---- if newfpos: print " new fp:", [e.tag for e in newfpos] ! if not options["TestDriver", "show_false_positives"]: newfpos = () for e in newfpos: *************** *** 294,298 **** if newfneg: print " new fn:", [e.tag for e in newfneg] ! if not options.show_false_negatives: newfneg = () for e in newfneg: --- 295,299 ---- if newfneg: print " new fn:", [e.tag for e in newfneg] ! if not options["TestDriver", "show_false_negatives"]: newfneg = () for e in newfneg: *************** *** 306,310 **** if newunsure: print " new unsure:", [e.tag for e in newunsure] ! if not options.show_unsure: newunsure = () for e in newunsure: --- 307,311 ---- if newunsure: print " new unsure:", [e.tag for e in newunsure] ! if not options["TestDriver", "show_unsure"]: newunsure = () for e in newunsure: *************** *** 313,317 **** printmsg(e, prob, clues) ! if options.show_histograms: printhist("this pair:", local_ham_hist, local_spam_hist) self.trained_ham_hist += local_ham_hist --- 314,318 ---- printmsg(e, prob, clues) ! if options["TestDriver", "show_histograms"]: printhist("this pair:", local_ham_hist, local_spam_hist) self.trained_ham_hist += local_ham_hist Index: classifier.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/classifier.py,v retrieving revision 1.8 retrieving revision 1.9 diff -C2 -d -r1.8 -r1.9 *** classifier.py 1 Sep 2003 21:07:32 -0000 1.8 --- classifier.py 5 Sep 2003 01:15:28 -0000 1.9 *************** *** 176,180 **** return prob ! if options.use_gary_combining: spamprob = gary_spamprob --- 176,180 ---- return prob ! if options["Classifier", "use_gary_combining"]: spamprob = gary_spamprob *************** *** 262,266 **** return prob ! if options.use_chi_squared_combining: spamprob = chi2_spamprob --- 262,266 ---- return prob ! if options["Classifier", "use_chi_squared_combining"]: spamprob = chi2_spamprob *************** *** 313,317 **** prob = spamratio / (hamratio + spamratio) ! if options.experimental_ham_spam_imbalance_adjustment: spam2ham = min(nspam / nham, 1.0) ham2spam = min(nham / nspam, 1.0) --- 313,317 ---- prob = spamratio / (hamratio + spamratio) ! if options["Classifier", "experimental_ham_spam_imbalance_adjustment"]: spam2ham = min(nspam / nham, 1.0) ham2spam = min(nham / nspam, 1.0) *************** *** 319,324 **** spam2ham = ham2spam = 1.0 ! S = options.unknown_word_strength ! StimesX = S * options.unknown_word_prob --- 319,324 ---- spam2ham = ham2spam = 1.0 ! S = options["Classifier", "unknown_word_strength"] ! StimesX = S * options["Classifier", "unknown_word_prob"] *************** *** 447,452 **** def _getclues(self, wordstream): ! mindist = options.minimum_prob_strength ! unknown = options.unknown_word_prob clues = [] # (distance, prob, word, record) tuples --- 447,452 ---- def _getclues(self, wordstream): ! mindist = options["Classifier", "minimum_prob_strength"] ! unknown = options["Classifier", "unknown_word_prob"] clues = [] # (distance, prob, word, record) tuples *************** *** 464,469 **** clues.sort() ! if len(clues) > options.max_discriminators: ! del clues[0 : -options.max_discriminators] # Return (prob, word, record). return [t[1:] for t in clues] --- 464,469 ---- clues.sort() ! if len(clues) > options["Classifier", "max_discriminators"]: ! del clues[0 : -options["Classifier", "max_discriminators"]] # Return (prob, word, record). return [t[1:] for t in clues] Index: dbmstorage.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/dbmstorage.py,v retrieving revision 1.7 retrieving revision 1.8 diff -C2 -d -r1.7 -r1.8 *** dbmstorage.py 7 Aug 2003 00:05:14 -0000 1.7 --- dbmstorage.py 5 Sep 2003 01:15:28 -0000 1.8 *************** *** 47,51 **** def open(*args): ! dbm_type = options.dbm_type.lower() f = open_funcs.get(dbm_type) if not f: --- 47,51 ---- def open(*args): ! dbm_type = options["globals", "dbm_type"].lower() f = open_funcs.get(dbm_type) if not f: Index: hammie.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/hammie.py,v retrieving revision 1.7 retrieving revision 1.8 diff -C2 -d -r1.7 -r1.8 *** hammie.py 26 Aug 2003 12:57:31 -0000 1.7 --- hammie.py 5 Sep 2003 01:15:28 -0000 1.8 *************** *** 44,49 **** for word, prob in clues if (word[0] == '*' or ! prob <= options.clue_mailheader_cutoff or ! prob >= 1.0 - options.clue_mailheader_cutoff)]) def score(self, msg, evidence=False): --- 44,51 ---- for word, prob in clues if (word[0] == '*' or ! prob <= options["Headers", ! "clue_mailheader_cutoff"] or ! prob >= 1.0 - options["Headers", ! "clue_mailheader_cutoff"])]) def score(self, msg, evidence=False): *************** *** 87,101 **** if header == None: ! header = options.hammie_header_name if spam_cutoff == None: ! spam_cutoff = options.spam_cutoff if ham_cutoff == None: ! ham_cutoff = options.ham_cutoff if debugheader == None: ! debugheader = options.hammie_debug_header_name if debug == None: ! debug = options.hammie_debug_header if train == None: ! train = options.hammie_train_on_filter msg = mboxutils.get_message(msg) --- 89,103 ---- if header == None: ! header = options["Headers", "classification_header_name"] if spam_cutoff == None: ! spam_cutoff = options["Categorization", "spam_cutoff"] if ham_cutoff == None: ! ham_cutoff = options["Categorization", "ham_cutoff"] if debugheader == None: ! debugheader = options["Hammie", "debug_header_name"] if debug == None: ! debug = options["Hammie", "debug_header"] if train == None: ! train = options["Hammie", "hammie_train_on_filter"] msg = mboxutils.get_message(msg) *************** *** 109,123 **** if prob < ham_cutoff: is_spam = False ! disp = options.header_ham_string elif prob > spam_cutoff: is_spam = True ! disp = options.header_spam_string else: is_spam = False ! disp = options.header_unsure_string if train: self.train(msg, is_spam, True) ! disp += ("; %."+str(options.header_score_digits)+"f") % prob ! if options.header_score_logarithm: if prob<=0.005 and prob>0.0: import math --- 111,125 ---- if prob < ham_cutoff: is_spam = False ! disp = options["Headers", "header_ham_string"] elif prob > spam_cutoff: is_spam = True ! disp = options["Headers", "header_spam_string"] else: is_spam = False ! disp = options["Headers", "header_unsure_string"] if train: self.train(msg, is_spam, True) ! disp += ("; %."+str(options["Headers", "header_score_digits"])+"f") % prob ! if options["Headers", "header_score_logarithm"]: if prob<=0.005 and prob>0.0: import math *************** *** 151,159 **** if add_header: if is_spam: ! trained = options.header_spam_string else: ! trained = options.header_ham_string ! del msg[options.hammie_trained_header] ! msg.add_header(options.hammie_trained_header, trained) def untrain(self, msg, is_spam): --- 153,161 ---- if add_header: if is_spam: ! trained = options["Headers", "header_spam_string"] else: ! trained = options["Headers", "header_ham_string"] ! del msg[options["Headers", "trained_header_name"]] ! msg.add_header(options["Headers", "trained_header_name"], trained) def untrain(self, msg, is_spam): *************** *** 181,195 **** msg = mboxutils.get_message(msg) ! trained = msg.get(options.hammie_trained_header) if not trained: return ! del msg[options.hammie_trained_header] ! if trained == options.header_ham_string: self.untrain_ham(msg) ! elif trained == options.header_spam_string: self.untrain_spam(msg) else: raise ValueError('%s header value unrecognized' ! % options.hammie_trained_header) def train_ham(self, msg, add_header=False): --- 183,197 ---- msg = mboxutils.get_message(msg) ! trained = msg.get(options["Headers", "trained_header_name"]) if not trained: return ! del msg[options["Headers", "trained_header_name"]] ! if trained == options["Headers", "header_ham_string"]: self.untrain_ham(msg) ! elif trained == options["Headers", "header_spam_string"]: self.untrain_spam(msg) else: raise ValueError('%s header value unrecognized' ! % options["Headers", "trained_header_name"]) def train_ham(self, msg, add_header=False): Index: storage.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v retrieving revision 1.27 retrieving revision 1.28 diff -C2 -d -r1.27 -r1.28 *** storage.py 1 Sep 2003 21:07:32 -0000 1.27 --- storage.py 5 Sep 2003 01:15:28 -0000 1.28 *************** *** 101,105 **** # tempbayes object is reclaimed when load() returns. ! if options.verbose: print >> sys.stderr, 'Loading state from',self.db_name,'pickle' --- 101,105 ---- # tempbayes object is reclaimed when load() returns. ! if options["globals", "verbose"]: print >> sys.stderr, 'Loading state from',self.db_name,'pickle' *************** *** 119,123 **** classifier.Classifier.__setstate__(self, tempbayes.__getstate__()) ! if options.verbose: print >> sys.stderr, ('%s is an existing pickle,' ' with %d ham and %d spam') \ --- 119,123 ---- classifier.Classifier.__setstate__(self, tempbayes.__getstate__()) ! if options["globals", "verbose"]: print >> sys.stderr, ('%s is an existing pickle,' ' with %d ham and %d spam') \ *************** *** 125,129 **** else: # new pickle ! if options.verbose: print >> sys.stderr, self.db_name,'is a new pickle' self.wordinfo = {} --- 125,129 ---- else: # new pickle ! if options["globals", "verbose"]: print >> sys.stderr, self.db_name,'is a new pickle' self.wordinfo = {} *************** *** 134,138 **** '''Store self as a pickle''' ! if options.verbose: print >> sys.stderr, 'Persisting',self.db_name,'as a pickle' --- 134,138 ---- '''Store self as a pickle''' ! if options["globals", "verbose"]: print >> sys.stderr, 'Persisting',self.db_name,'as a pickle' *************** *** 160,164 **** '''Load state from database''' ! if options.verbose: print >> sys.stderr, 'Loading state from',self.db_name,'database' --- 160,164 ---- '''Load state from database''' ! if options["globals", "verbose"]: print >> sys.stderr, 'Loading state from',self.db_name,'database' *************** *** 172,176 **** (self.nspam, self.nham) = t[1:] ! if options.verbose: print >> sys.stderr, ('%s is an existing database,' ' with %d spam and %d ham') \ --- 172,176 ---- (self.nspam, self.nham) = t[1:] ! if options["globals", "verbose"]: print >> sys.stderr, ('%s is an existing database,' ' with %d spam and %d ham') \ *************** *** 178,182 **** else: # new database ! if options.verbose: print >> sys.stderr, self.db_name,'is a new database' self.nspam = 0 --- 178,182 ---- else: # new database ! if options["globals", "verbose"]: print >> sys.stderr, self.db_name,'is a new database' self.nspam = 0 *************** *** 188,192 **** '''Place state into persistent store''' ! if options.verbose: print >> sys.stderr, 'Persisting',self.db_name,'state in database' --- 188,192 ---- '''Place state into persistent store''' ! if options["globals", "verbose"]: print >> sys.stderr, 'Persisting',self.db_name,'state in database' *************** *** 424,428 **** import psycopg ! if options.verbose: print >> sys.stderr, 'Loading state from',self.db_name,'database' --- 424,428 ---- import psycopg ! if options["globals", "verbose"]: print >> sys.stderr, 'Loading state from',self.db_name,'database' *************** *** 440,444 **** self.nspam = row["nspam"] self.nham = row["nham"] ! if options.verbose: print >> sys.stderr, ('%s is an existing database,' ' with %d spam and %d ham') \ --- 440,444 ---- self.nspam = row["nspam"] self.nham = row["nham"] ! if options["globals", "verbose"]: print >> sys.stderr, ('%s is an existing database,' ' with %d spam and %d ham') \ *************** *** 446,450 **** else: # new database ! if options.verbose: print >> sys.stderr, self.db_name,'is a new database' self.nspam = 0 --- 446,450 ---- else: # new database ! if options["globals", "verbose"]: print >> sys.stderr, self.db_name,'is a new database' self.nspam = 0 *************** *** 495,499 **** import MySQLdb ! if options.verbose: print >> sys.stderr, 'Loading state from',self.db_name,'database' --- 495,499 ---- import MySQLdb ! if options["globals", "verbose"]: print >> sys.stderr, 'Loading state from',self.db_name,'database' *************** *** 512,516 **** self.nspam = int(row[1]) self.nham = int(row[2]) ! if options.verbose: print >> sys.stderr, ('%s is an existing database,' ' with %d spam and %d ham') \ --- 512,516 ---- self.nspam = int(row[1]) self.nham = int(row[2]) ! if options["globals", "verbose"]: print >> sys.stderr, ('%s is an existing database,' ' with %d spam and %d ham') \ *************** *** 518,522 **** else: # new database ! if options.verbose: print >> sys.stderr, self.db_name,'is a new database' self.nspam = 0 --- 518,522 ---- else: # new database ! if options["globals", "verbose"]: print >> sys.stderr, self.db_name,'is a new database' self.nspam = 0 *************** *** 555,559 **** '''Train the database with the message''' ! if options.verbose: print >> sys.stderr, 'training with',message.key() --- 555,559 ---- '''Train the database with the message''' ! if options["globals", "verbose"]: print >> sys.stderr, 'training with',message.key() *************** *** 569,573 **** '''Untrain the database with the message''' ! if options.verbose: print >> sys.stderr, 'untraining with',message.key() --- 569,573 ---- '''Untrain the database with the message''' ! if options["globals", "verbose"]: print >> sys.stderr, 'untraining with',message.key() Index: tokenizer.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/tokenizer.py,v retrieving revision 1.13 retrieving revision 1.14 diff -C2 -d -r1.13 -r1.14 *** tokenizer.py 28 Jun 2003 01:51:57 -0000 1.13 --- tokenizer.py 5 Sep 2003 01:15:28 -0000 1.14 *************** *** 657,661 **** yield "fname piece:" + piece ! def tokenize_word(word, _len=len, maxword=options.skip_max_word_size): n = _len(word) # Make sure this range matches in tokenize(). --- 657,662 ---- yield "fname piece:" + piece ! def tokenize_word(word, _len=len, maxword=options["Tokenizer", ! "skip_max_word_size"]): n = _len(word) # Make sure this range matches in tokenize(). *************** *** 680,684 **** # XXX Figure out why, and/or see if some other way of summarizing # XXX this info has greater benefit. ! if options.generate_long_skips: yield "skip:%c %d" % (word[0], n // 10 * 10) if has_highbit_char(word): --- 681,685 ---- # XXX Figure out why, and/or see if some other way of summarizing # XXX this info has greater benefit. ! if options["Tokenizer", "generate_long_skips"]: yield "skip:%c %d" % (word[0], n // 10 * 10) if has_highbit_char(word): *************** *** 1070,1076 **** def __init__(self): ! if options.basic_header_tokenize: self.basic_skip = [re.compile(s) ! for s in options.basic_header_skip] def get_message(self, obj): --- 1071,1078 ---- def __init__(self): ! if options["Tokenizer", "basic_header_tokenize"]: self.basic_skip = [re.compile(s) ! for s in options["Tokenizer", ! "basic_header_skip"]] def get_message(self, obj): *************** *** 1115,1119 **** # the best discriminators. # (Not just Date, but Received and X-From_.) ! if options.basic_header_tokenize: for k, v in msg.items(): k = k.lower() --- 1117,1121 ---- # the best discriminators. # (Not just Date, but Received and X-From_.) ! if options["Tokenizer", "basic_header_tokenize"]: for k, v in msg.items(): k = k.lower() *************** *** 1126,1130 **** for t in tokenize_word(w): yield "%s:%s" % (k, t) ! if options.basic_header_tokenize_only: return --- 1128,1132 ---- for t in tokenize_word(w): yield "%s:%s" % (k, t) ! if options["Tokenizer", "basic_header_tokenize_only"]: return *************** *** 1157,1161 **** # To:, Cc: # These can help, if your ham and spam are sourced # # from the same location. If not, they'll be horrible. ! for field in options.address_headers: addrlist = msg.get_all(field, []) if not addrlist: --- 1159,1163 ---- # To:, Cc: # These can help, if your ham and spam are sourced # # from the same location. If not, they'll be horrible. ! for field in options["Tokenizer", "address_headers"]: addrlist = msg.get_all(field, []) if not addrlist: *************** *** 1199,1203 **** # eliminates the bad case where the message was sent to a single # individual. ! if options.summarize_email_prefixes: all_addrs = [] addresses = msg.get_all('to', []) + msg.get_all('cc', []) --- 1201,1205 ---- # eliminates the bad case where the message was sent to a single # individual. ! if options["Tokenizer", "summarize_email_prefixes"]: all_addrs = [] addresses = msg.get_all('to', []) + msg.get_all('cc', []) *************** *** 1226,1230 **** # , , # , , ! if options.summarize_email_suffixes: all_addrs = [] addresses = msg.get_all('to', []) + msg.get_all('cc', []) --- 1228,1232 ---- # , , # , , ! if options["Tokenizer", "summarize_email_suffixes"]: all_addrs = [] addresses = msg.get_all('to', []) + msg.get_all('cc', []) *************** *** 1273,1277 **** # Received: # Neil Schemenauer reports good results from this. ! if options.mine_received_headers: for header in msg.get_all("received", ()): for pat, breakdown in [(received_host_re, breakdown_host), --- 1275,1279 ---- # Received: # Neil Schemenauer reports good results from this. ! if options["Tokenizer", "mine_received_headers"]: for header in msg.get_all("received", ()): for pat, breakdown in [(received_host_re, breakdown_host), *************** *** 1283,1287 **** # Date: ! if options.generate_time_buckets: for header in msg.get_all("date", ()): mat = self.date_hms_re.search(header) --- 1285,1289 ---- # Date: ! if options["Tokenizer", "generate_time_buckets"]: for header in msg.get_all("date", ()): mat = self.date_hms_re.search(header) *************** *** 1293,1297 **** yield 'time:%02d:%d' % (h, bucket) ! if options.extract_dow: for header in msg.get_all("date", ()): # extract the day of the week --- 1295,1299 ---- yield 'time:%02d:%d' % (h, bucket) ! if options["Tokenizer", "extract_dow"]: for header in msg.get_all("date", ()): # extract the day of the week *************** *** 1324,1328 **** # X-Complaints-To a strong ham clue. x2n = {} ! if options.count_all_header_lines: for x in msg.keys(): x2n[x] = x2n.get(x, 0) + 1 --- 1326,1330 ---- # X-Complaints-To a strong ham clue. x2n = {} ! if options["Tokenizer", "count_all_header_lines"]: for x in msg.keys(): x2n[x] = x2n.get(x, 0) + 1 *************** *** 1343,1347 **** yield "noheader:" + k ! def tokenize_body(self, msg, maxword=options.skip_max_word_size): """Generate a stream of tokens from an email Message. --- 1345,1350 ---- yield "noheader:" + k ! def tokenize_body(self, msg, maxword=options["Tokenizer", ! "skip_max_word_size"]): """Generate a stream of tokens from an email Message. *************** *** 1350,1354 **** """ ! if options.check_octets: # Find, decode application/octet-stream parts of the body, # tokenizing the first few characters of each chunk. --- 1353,1357 ---- """ ! if options["Tokenizer", "check_octets"]: # Find, decode application/octet-stream parts of the body, # tokenizing the first few characters of each chunk. *************** *** 1364,1368 **** continue ! yield "octet:%s" % text[:options.octet_prefix_size] # Find, decode (base64, qp), and tokenize textual parts of the body. --- 1367,1372 ---- continue ! yield "octet:%s" % text[:options["Tokenizer", ! "octet_prefix_size"]] # Find, decode (base64, qp), and tokenize textual parts of the body. *************** *** 1388,1392 **** text = text.lower() ! if options.replace_nonascii_chars: # Replace high-bit chars and control chars with '?'. text = text.translate(non_ascii_translate_tab) --- 1392,1396 ---- text = text.lower() ! if options["Tokenizer", "replace_nonascii_chars"]: # Replace high-bit chars and control chars with '?'. text = text.translate(non_ascii_translate_tab) From anadelonbrin at users.sourceforge.net Thu Sep 4 19:16:48 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Thu Sep 4 21:16:52 2003 Subject: [Spambayes-checkins] spambayes/testtools pop3proxytest.py,1.1,NONE Message-ID: Update of /cvsroot/spambayes/spambayes/testtools In directory sc8-pr-cvs1:/tmp/cvs-serv14316/testtools Removed Files: pop3proxytest.py Log Message: Crap! We can't use "sb-" as a prefix, because then we can't import the scripts. I guess that all the importable code could be moved into modules, but that seems like a huge hassle. Let's use "sb_" as a prefix instead. Apologies for cluttering the attic...sigh. --- pop3proxytest.py DELETED --- From anadelonbrin at users.sourceforge.net Thu Sep 4 19:16:48 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Thu Sep 4 21:16:57 2003 Subject: [Spambayes-checkins] spambayes/spambayes/test test_sb-server.py, NONE, 1.1 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes/test In directory sc8-pr-cvs1:/tmp/cvs-serv14316/spambayes/test Added Files: test_sb-server.py Log Message: Crap! We can't use "sb-" as a prefix, because then we can't import the scripts. I guess that all the importable code could be moved into modules, but that seems like a huge hassle. Let's use "sb_" as a prefix instead. Apologies for cluttering the attic...sigh. --- NEW FILE: test_sb-server.py --- #! /usr/bin/env python """Test the pop3proxy is working correctly. When using the -z command line option, carries out a test that the POP3 proxy can be connected to, that incoming mail is classified, that pipelining is removed from the CAPA[bility] query, and that the web ui is present. The -t option runs a fake POP3 server on port 8810. This is the same server that the -z option uses, and may be separately run for other testing purposes. Usage: pop3proxytest.py [options] options: -z : Runs a self-test and exits. -t : Runs a fake POP3 server on port 8110 (for testing). -h : Displays this help message. """ # This module is part of the spambayes project, which is Copyright 2002 # The Python Software Foundation and is covered by the Python Software # Foundation license. __author__ = "Richie Hindle " __credits__ = "All the Spambayes folk." try: True, False except NameError: # Maintain compatibility with Python 2.2 True, False = 1, 0 # This code originally formed a part of pop3proxy.py. If you are examining # the history of this file, you may need to go back to there. todo = """ Web training interface: o Functional tests. """ # One example of spam and one of ham - both are used to train, and are # then classified. Not a good test of the classifier, but a perfectly # good test of the POP3 proxy. The bodies of these came from the # spambayes project, and Richie added the headers because the # originals had no headers. spam1 = """From: friend@public.com Subject: Make money fast Hello tim_chandler , Want to save money ? Now is a good time to consider refinancing. Rates are low so you can cut your current payments and save money. http://64.251.22.101/interest/index%38%30%300%2E%68t%6D Take off list on site [s5] """ good1 = """From: chris@example.com Subject: ZPT and DTML Jean Jordaan wrote: > 'Fraid so ;> It contains a vintage dtml-calendar tag. > http://www.zope.org/Members/teyc/CalendarTag > > Hmm I think I see what you mean: one needn't manually pass on the > namespace to a ZPT? Yeah, Page Templates are a bit more clever, sadly, DTML methods aren't :-( Chris """ import asyncore import socket import operator import re import getopt # a bit of a hack to help those without spambayes on their # Python path - stolen from timtest.py import sys import os sys.path.insert(-1, os.getcwd()) sys.path.insert(-1, os.path.dirname(os.getcwd())) from spambayes import Dibbler from spambayes import tokenizer from spambayes.UserInterface import UserInterfaceServer from spambayes.ProxyUI import ProxyUserInterface from pop3proxy import BayesProxyListener from pop3proxy import state, _recreateState from spambayes.Options import options # HEADER_EXAMPLE is the longest possible header - the length of this one # is added to the size of each message. HEADER_EXAMPLE = '%s: xxxxxxxxxxxxxxxxxxxx\r\n' % \ options["Hammie", "header_name"] class TestListener(Dibbler.Listener): """Listener for TestPOP3Server. Works on port 8110, to co-exist with real POP3 servers.""" def __init__(self, socketMap=asyncore.socket_map): Dibbler.Listener.__init__(self, 8110, TestPOP3Server, (socketMap,), socketMap=socketMap) class TestPOP3Server(Dibbler.BrighterAsyncChat): """Minimal POP3 server, for testing purposes. Doesn't support UIDL. USER, PASS, APOP, DELE and RSET simply return "+OK" without doing anything. Also understands the 'KILL' command, to kill it. The mail content is the example messages above. """ def __init__(self, clientSocket, socketMap): # Grumble: asynchat.__init__ doesn't take a 'map' argument, # hence the two-stage construction. Dibbler.BrighterAsyncChat.__init__(self) Dibbler.BrighterAsyncChat.set_socket(self, clientSocket, socketMap) self.maildrop = [spam1, good1] self.set_terminator('\r\n') self.okCommands = ['USER', 'PASS', 'APOP', 'NOOP', 'DELE', 'RSET', 'QUIT', 'KILL'] self.handlers = {'CAPA': self.onCapa, 'STAT': self.onStat, 'LIST': self.onList, 'RETR': self.onRetr, 'TOP': self.onTop} self.push("+OK ready\r\n") self.request = '' def collect_incoming_data(self, data): """Asynchat override.""" self.request = self.request + data def found_terminator(self): """Asynchat override.""" if ' ' in self.request: command, args = self.request.split(None, 1) else: command, args = self.request, '' command = command.upper() if command in self.okCommands: self.push("+OK (we hope)\r\n") if command == 'QUIT': self.close_when_done() if command == 'KILL': self.socket.shutdown(2) self.close() raise SystemExit else: handler = self.handlers.get(command, self.onUnknown) self.push(handler(command, args)) # Or push_slowly for testing self.request = '' def push_slowly(self, response): """Useful for testing.""" for c in response: self.push(c) time.sleep(0.02) def onCapa(self, command, args): """POP3 CAPA command. This lies about supporting pipelining for test purposes - the POP3 proxy *doesn't* support pipelining, and we test that it correctly filters out that capability from the proxied capability list.""" lines = ["+OK Capability list follows", "PIPELINING", "TOP", ".", ""] return '\r\n'.join(lines) def onStat(self, command, args): """POP3 STAT command.""" maildropSize = reduce(operator.add, map(len, self.maildrop)) maildropSize += len(self.maildrop) * len(HEADER_EXAMPLE) return "+OK %d %d\r\n" % (len(self.maildrop), maildropSize) def onList(self, command, args): """POP3 LIST command, with optional message number argument.""" if args: try: number = int(args) except ValueError: number = -1 if 0 < number <= len(self.maildrop): return "+OK %d\r\n" % len(self.maildrop[number-1]) else: return "-ERR no such message\r\n" else: returnLines = ["+OK"] for messageIndex in range(len(self.maildrop)): size = len(self.maildrop[messageIndex]) returnLines.append("%d %d" % (messageIndex + 1, size)) returnLines.append(".") return '\r\n'.join(returnLines) + '\r\n' def _getMessage(self, number, maxLines): """Implements the POP3 RETR and TOP commands.""" if 0 < number <= len(self.maildrop): message = self.maildrop[number-1] headers, body = message.split('\n\n', 1) bodyLines = body.split('\n')[:maxLines] message = headers + '\r\n\r\n' + '\n'.join(bodyLines) return "+OK\r\n%s\r\n.\r\n" % message else: return "-ERR no such message\r\n" def onRetr(self, command, args): """POP3 RETR command.""" try: number = int(args) except ValueError: number = -1 return self._getMessage(number, 12345) def onTop(self, command, args): """POP3 RETR command.""" try: number, lines = map(int, args.split()) except ValueError: number, lines = -1, -1 return self._getMessage(number, lines) def onUnknown(self, command, args): """Unknown POP3 command.""" return "-ERR Unknown command: %s\r\n" % repr(command) def test(): """Runs a self-test using TestPOP3Server, a minimal POP3 server that serves the example emails above. """ # Run a proxy and a test server in separate threads with separate # asyncore environments. import threading state.isTest = True testServerReady = threading.Event() def runTestServer(): testSocketMap = {} TestListener(socketMap=testSocketMap) testServerReady.set() asyncore.loop(map=testSocketMap) proxyReady = threading.Event() def runUIAndProxy(): httpServer = UserInterfaceServer(8881) proxyUI = ProxyUserInterface(state, _recreateState) httpServer.register(proxyUI) BayesProxyListener('localhost', 8110, ('', 8111)) state.bayes.learn(tokenizer.tokenize(spam1), True) state.bayes.learn(tokenizer.tokenize(good1), False) proxyReady.set() Dibbler.run() threading.Thread(target=runTestServer).start() testServerReady.wait() threading.Thread(target=runUIAndProxy).start() proxyReady.wait() # Connect to the proxy and the test server. proxy = socket.socket(socket.AF_INET, socket.SOCK_STREAM) proxy.connect(('localhost', 8111)) response = proxy.recv(100) assert response == "+OK ready\r\n" pop3Server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) pop3Server.connect(('localhost', 8110)) response = pop3Server.recv(100) assert response == "+OK ready\r\n" # Verify that the test server claims to support pipelining. pop3Server.send("capa\r\n") response = pop3Server.recv(1000) assert response.find("PIPELINING") >= 0 # Ask for the capabilities via the proxy, and verify that the proxy # is filtering out the PIPELINING capability. proxy.send("capa\r\n") response = proxy.recv(1000) assert response.find("PIPELINING") == -1 # Stat the mailbox to get the number of messages. proxy.send("stat\r\n") response = proxy.recv(100) count, totalSize = map(int, response.split()[1:3]) assert count == 2 # Loop through the messages ensuring that they have judgement # headers. for i in range(1, count+1): response = "" proxy.send("retr %d\r\n" % i) while response.find('\n.\r\n') == -1: response = response + proxy.recv(1000) assert response.find(options["Hammie", "header_name"]) >= 0 # Smoke-test the HTML UI. httpServer = socket.socket(socket.AF_INET, socket.SOCK_STREAM) httpServer.connect(('localhost', 8881)) httpServer.sendall("get / HTTP/1.0\r\n\r\n") response = '' while 1: packet = httpServer.recv(1000) if not packet: break response += packet assert re.search(r"(?s).*Spambayes proxy.*", response) # Kill the proxy and the test server. proxy.sendall("kill\r\n") proxy.recv(100) pop3Server.sendall("kill\r\n") pop3Server.recv(100) def run(): # Read the arguments. try: opts, args = getopt.getopt(sys.argv[1:], 'htz') except getopt.error, msg: print >>sys.stderr, str(msg) + '\n\n' + __doc__ sys.exit() runSelfTest = False for opt, arg in opts: if opt == '-h': print >>sys.stderr, __doc__ sys.exit() elif opt == '-t': state.isTest = True state.runTestServer = True elif opt == '-z': state.isTest = True runSelfTest = True state.createWorkers() if runSelfTest: print "\nRunning self-test...\n" state.buildServerStrings() test() print "Self-test passed." # ...else it would have asserted. elif state.runTestServer: print "Running a test POP3 server on port 8110..." TestListener() asyncore.loop() else: print >>sys.stderr, __doc__ if __name__ == '__main__': run() From anadelonbrin at users.sourceforge.net Thu Sep 4 19:16:48 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Thu Sep 4 21:19:37 2003 Subject: [Spambayes-checkins] spambayes/scripts sb_client.py, NONE, 1.1 sb_dbexpimp.py, NONE, 1.1 sb_filter.py, NONE, 1.1 sb_imapfilter.py, NONE, 1.1 sb_mailsort.py, NONE, 1.1 sb_mboxtrain.py, NONE, 1.1 sb_notesfilter.py, NONE, 1.1 sb_pop3dnd.py, NONE, 1.1 sb_server.py, NONE, 1.1 sb_smtpproxy.py, NONE, 1.1 sb_unheader.py, NONE, 1.1 sb_upload.py, NONE, 1.1 sb_xmlrpcserver.py, NONE, 1.1 sb-client.py, 1.1, NONE sb-dbexpimp.py, 1.1, NONE sb-filter.py, 1.1, NONE sb-imapfilter.py, 1.1, NONE sb-mailsort.py, 1.1, NONE sb-mboxtrain.py, 1.1, NONE sb-notesfilter.py, 1.1, NONE sb-pop3dnd.py, 1.1, NONE sb-server.py, 1.1, NONE sb-smtpproxy.py, 1.1, NONE sb-unheader.py, 1.1, NONE sb-upload.py, 1.1, NONE sb-xmlrpcserver.py, 1.1, NONE Message-ID: Update of /cvsroot/spambayes/spambayes/scripts In directory sc8-pr-cvs1:/tmp/cvs-serv14316/scripts Added Files: sb_client.py sb_dbexpimp.py sb_filter.py sb_imapfilter.py sb_mailsort.py sb_mboxtrain.py sb_notesfilter.py sb_pop3dnd.py sb_server.py sb_smtpproxy.py sb_unheader.py sb_upload.py sb_xmlrpcserver.py Removed Files: sb-client.py sb-dbexpimp.py sb-filter.py sb-imapfilter.py sb-mailsort.py sb-mboxtrain.py sb-notesfilter.py sb-pop3dnd.py sb-server.py sb-smtpproxy.py sb-unheader.py sb-upload.py sb-xmlrpcserver.py Log Message: Crap! We can't use "sb-" as a prefix, because then we can't import the scripts. I guess that all the importable code could be moved into modules, but that seems like a huge hassle. Let's use "sb_" as a prefix instead. Apologies for cluttering the attic...sigh. --- NEW FILE: sb_client.py --- #! /usr/bin/env python """A client for hammiesrv. Just feed it your mail on stdin, and it spits out the same message with the spambayes score in a new X-Spambayes-Disposition header. """ import xmlrpclib import sys RPCBASE="http://localhost:65000" def main(): msg = sys.stdin.read() try: x = xmlrpclib.ServerProxy(RPCBASE) m = xmlrpclib.Binary(msg) out = x.filter(m) print out.data except: if __debug__: import traceback traceback.print_exc() print msg if __name__ == "__main__": main() --- NEW FILE: sb_dbexpimp.py --- #! /usr/bin/env python """dbExpImp.py - Bayes database export/import Classes: Abstract: This utility has the primary function of exporting and importing a spambayes database into/from a flat file. This is useful in a number of scenarios. Platform portability of database - flat files can be exported and imported across platforms (winduhs and linux, for example) Database implementation changes - databases can survive database implementation upgrades or new database implementations. For example, if a dbm implementation changes between python x.y and python x.y+1... Database reorganization - an export followed by an import reorgs an existing database, improving performance, at least in some database implementations Database sharing - it is possible to distribute particular databases for research purposes, database sharing purposes, or for new users to have a 'seed' database to start with. Database merging - multiple databases can be merged into one quite easily by simply not specifying -n on an import. This will add the two database nham and nspams together (assuming the two databases do not share corpora) and for wordinfo conflicts, will add spamcount and hamcount together. Spambayes software release migration - an export can be executed before a release upgrade, as part of the installation script. Then, after the new software is installed, an import can be executed, which will effectively preserve existing training. This eliminates the need for retraining every time a release is installed. Others? I'm sure I haven't thought of everything... Usage: dbExpImp [options] options: -e : export -i : import -v : verbose mode (some additional diagnostic messages) -f: FN : flat file to export to or import from -d: FN : name of pickled database file to use -D: FN : name of dbm database file to use -m : merge import into an existing database file. This is meaningful only for import. If omitted, a new database file will be created. If specified, the imported wordinfo will be merged into an existing database. Run dbExpImp -h for more information. -h : help Examples: Export pickled mybayes.db into mybayes.db.export as a csv flat file dbExpImp -e -d mybayes.db -f mybayes.db.export Import mybayes.eb.export into a new DBM mybayes.db dbExpImp -i -D mybayes.db -f mybayes.db.export Export, then import (reorganize) new pickled mybayes.db dbExpImp -e -i -n -d mybayes.db -f mybayes.db.export Convert a bayes database from pickle to DBM dbExpImp -e -d abayes.db -f abayes.export dbExpImp -i -D abayes.db -f abayes.export Create a new database (newbayes.db) from two databases (abayes.db, bbayes.db) dbExpImp -e -d abayes.db -f abayes.export dbExpImp -e -d bbayes.db -f bbayes.export dbExpImp -i -d newbayes.db -f abayes.export dbExpImp -i -m -d newbayes.db -f bbayes.export To Do: o Suggestions? """ # This module is part of the spambayes project, which is Copyright 2002 # The Python Software Foundation and is covered by the Python Software # Foundation license. __author__ = "Tim Stone " from __future__ import generators try: True, False except NameError: # Maintain compatibility with Python 2.2 True, False = 1, 0 import spambayes.storage from spambayes.Options import options import sys, os, getopt, errno, re import urllib def runExport(dbFN, useDBM, outFN): if useDBM: bayes = spambayes.storage.DBDictClassifier(dbFN) words = bayes.db.keys() words.remove(bayes.statekey) else: bayes = spambayes.storage.PickledClassifier(dbFN) words = bayes.wordinfo.keys() try: fp = open(outFN, 'w') except IOError, e: if e.errno != errno.ENOENT: raise nham = bayes.nham; nspam = bayes.nspam; print "Exporting database %s to file %s" % (dbFN, outFN) print "Database has %s ham, %s spam, and %s words" \ % (nham, nspam, len(words)) fp.write("%s,%s,\n" % (nham, nspam)) for word in words: wi = bayes._wordinfoget(word) hamcount = wi.hamcount spamcount = wi.spamcount word = urllib.quote(word) fp.write("%s`%s`%s`\n" % (word, hamcount, spamcount)) fp.close() def runImport(dbFN, useDBM, newDBM, inFN): if newDBM: try: os.unlink(dbFN) except OSError, e: if e.errno != 2: # errno. raise try: os.unlink(dbFN+".dat") except OSError, e: if e.errno != 2: # errno. raise try: os.unlink(dbFN+".dir") except OSError, e: if e.errno != 2: # errno. raise if useDBM: bayes = spambayes.storage.DBDictClassifier(dbFN) else: bayes = spambayes.storage.PickledClassifier(dbFN) try: fp = open(inFN, 'r') except IOError, e: if e.errno != errno.ENOENT: raise nline = fp.readline() (nham, nspam, junk) = re.split(',', nline) if newDBM: bayes.nham = int(nham) bayes.nspam = int(nspam) else: bayes.nham += int(nham) bayes.nspam += int(nspam) if newDBM: impType = "Importing" else: impType = "Merging" print "%s database %s using file %s" % (impType, dbFN, inFN) lines = fp.readlines() for line in lines: (word, hamcount, spamcount, junk) = re.split('`', line) word = urllib.unquote(word) try: wi = bayes.wordinfo[word] except KeyError: wi = bayes.WordInfoClass() wi.hamcount += int(hamcount) wi.spamcount += int(spamcount) bayes._wordinfoset(word, wi) fp.close() print "Storing database, please be patient. Even moderately large" print "databases may take a very long time to store." bayes.store() print "Finished storing database" if useDBM: words = bayes.db.keys() words.remove(bayes.statekey) else: words = bayes.wordinfo.keys() print "Database has %s ham, %s spam, and %s words" \ % (bayes.nham, bayes.nspam, len(words)) if __name__ == '__main__': try: opts, args = getopt.getopt(sys.argv[1:], 'iehmvd:D:f:') except getopt.error, msg: print >>sys.stderr, str(msg) + '\n\n' + __doc__ sys.exit() usePickle = False useDBM = False newDBM = True dbFN = None flatFN = None exp = False imp = False for opt, arg in opts: if opt == '-h': print >>sys.stderr, __doc__ sys.exit() elif opt == '-d': useDBM = False dbFN = arg elif opt == '-D': useDBM = True dbFN = arg elif opt == '-f': flatFN = arg elif opt == '-e': exp = True elif opt == '-i': imp = True elif opt == '-m': newDBM = False elif opt == '-v': options["globals", "verbose"] = True if (dbFN and flatFN): if exp: runExport(dbFN, useDBM, flatFN) if imp: runImport(dbFN, useDBM, newDBM, flatFN) else: print >>sys.stderr, __doc__ --- NEW FILE: sb_filter.py --- #!/usr/bin/env python ## A hammie front-end to make the simple stuff simple. ## ## ## The intent is to call this from procmail and its ilk like so: ## ## :0 fw ## | hammiefilter.py ## ## Then, you can set up your MUA to pipe ham and spam to it, one at a ## time, by calling it with either the -g or -s options, respectively. ## ## Author: Neale Pickett ## """Usage: %(program)s [OPTION]... [OPTION] is one of: -h show usage and exit -x show some usage examples and exit -d DBFILE use database in DBFILE -D PICKLEFILE use pickle (instead of database) in PICKLEFILE -n create a new database *+ -f filter (default if no processing options are given) *+ -t [EXPERIMENTAL] filter and train based on the result (you must make sure to untrain all mistakes later) * -g [EXPERIMENTAL] (re)train as a good (ham) message * -s [EXPERIMENTAL] (re)train as a bad (spam) message * -G [EXPERIMENTAL] untrain ham (only use if you've already trained this message) * -S [EXPERIMENTAL] untrain spam (only use if you've already trained this message) All options marked with '*' operate on stdin. Only those processing options marked with '+' send a modified message to stdout. """ import os import sys import getopt from spambayes import hammie, Options, mboxutils # See Options.py for explanations of these properties program = sys.argv[0] example_doc = """_Examples_ filter a message on disk: %(program)s < message (re)train a message as ham: %(program)s -g < message (re)train a message as spam: %(program)s -s < message procmail recipe to filter and train in one step: :0 fw | %(program)s -t mutt configuration. This binds the 'H' key to retrain the message as ham, and prompt for a folder to move it to. The 'S' key retrains as spam, and moves to a 'spam' folder. XXX: add this """ def examples(): print example_doc % globals() sys.exit(0) def usage(code, msg=''): """Print usage message and sys.exit(code).""" if msg: print >> sys.stderr, msg print >> sys.stderr print >> sys.stderr, __doc__ % globals() sys.exit(code) class HammieFilter(object): def __init__(self): options = Options.options # This is a bit of a hack to counter the default for # persistent_storage_file changing from ~/.hammiedb to hammie.db # This will work unless a user: # * had hammie.db as their value for persistent_storage_file, and # * their config file was loaded by Options.py. if options["Storage", "persistent_storage_file"] == \ options.default("Storage", "persistent_storage_file"): options["Storage", "persistent_storage_file"] = \ "~/.hammiedb" options.merge_files(['/etc/hammierc', os.path.expanduser('~/.hammierc')]) self.dbname = options["Storage", "persistent_storage_file"] self.dbname = os.path.expanduser(self.dbname) self.usedb = options["Storage", "persistent_use_database"] def newdb(self): h = hammie.open(self.dbname, self.usedb, 'n') h.store() print >> sys.stderr, "Created new database in", self.dbname def filter(self, msg): h = hammie.open(self.dbname, self.usedb, 'r') return h.filter(msg) def filter_train(self, msg): h = hammie.open(self.dbname, self.usedb, 'c') return h.filter(msg, train=True) def train_ham(self, msg): h = hammie.open(self.dbname, self.usedb, 'c') h.train_ham(msg, True) h.store() def train_spam(self, msg): h = hammie.open(self.dbname, self.usedb, 'c') h.train_spam(msg, True) h.store() def untrain_ham(self, msg): h = hammie.open(self.dbname, self.usedb, 'c') h.untrain_ham(msg) h.store() def untrain_spam(self, msg): h = hammie.open(self.dbname, self.usedb, 'c') h.untrain_spam(msg) h.store() def main(): h = HammieFilter() actions = [] opts, args = getopt.getopt(sys.argv[1:], 'hxd:D:nfgstGS', ['help', 'examples']) for opt, arg in opts: if opt in ('-h', '--help'): usage(0) elif opt in ('-x', '--examples'): examples() elif opt == '-d': h.usedb = True h.dbname = arg elif opt == '-D': h.usedb = False h.dbname = arg elif opt == '-f': actions.append(h.filter) elif opt == '-g': actions.append(h.train_ham) elif opt == '-s': actions.append(h.train_spam) elif opt == '-t': actions.append(h.filter_train) elif opt == '-G': actions.append(h.untrain_ham) elif opt == '-S': actions.append(h.untrain_spam) elif opt == "-n": h.newdb() sys.exit(0) if actions == []: actions = [h.filter] msg = mboxutils.get_message(sys.stdin) for action in actions: action(msg) sys.stdout.write(msg.as_string(unixfrom=(msg.get_unixfrom() is not None))) if __name__ == "__main__": main() --- NEW FILE: sb_imapfilter.py --- #!/usr/bin/env python """An IMAP filter. An IMAP message box is scanned and all non-scored messages are scored and (where necessary) filtered. The original filter design owed much to isbg by Roger Binns (http://www.rogerbinns.com/isbg). Usage: imapfilter [options] note: option values with spaces in them must be enclosed in double quotes options: -d dbname : pickled training database filename -D dbname : dbm training database filename -t : train contents of spam folder and ham folder -c : classify inbox -h : help -v : verbose mode -p : security option to prompt for imap password, rather than look in options["imap", "password"] -e y/n : expunge/purge messages on exit (y) or not (n) -i debuglvl : a somewhat mysterious imaplib debugging level -l minutes : period of time between filtering operations -b : Launch a web browser showing the user interface. Examples: Classify inbox, with dbm database imapfilter -c -D bayes.db Train Spam and Ham, then classify inbox, with dbm database imapfilter -t -c -D bayes.db Train Spam and Ham only, with pickled database imapfilter -t -d bayes.db Warnings: o This is alpha software! The filter is currently being developed and tested. We do *not* recommend using it on a production system unless you are confident that you can get your mail back if you lose it. On the other hand, we do recommend that you test it for us and let us know if anything does go wrong. o By default, the filter does *not* delete, modify or move any of your mail. Due to quirks in how imap works, new versions of your mail are modified and placed in new folders, but the originals are still available. These are flagged with the /Deleted flag so that you know that they can be removed. Your mailer may not show these messages by default, but there should be an option to do so. *However*, if your mailer automatically purges/expunges (i.e. permanently deletes) mail flagged as such, *or* if you set the imap_expunge option to True, then this mail will be irretrievably lost. To Do: o IMAPMessage and IMAPFolder currently carry out very simple checks of responses received from IMAP commands, but if the response is not "OK", then the filter terminates. Handling of these errors could be much nicer. o IMAP over SSL is untested. o Develop a test script, like spambayes/test/test_pop3proxy.py that runs through some tests (perhaps with a *real* imap server, rather than a dummy one). This would make it easier to carry out the tests against each server whenever a change is made. o IMAP supports authentication via other methods than the plain-text password method that we are using at the moment. Neither of the servers I have access to offer any alternative method, however. If someone's does, then it would be nice to offer this. o Usernames should be able to be literals as well as quoted strings. This might help if the username/password has special characters like accented characters. o Suggestions? """ # This module is part of the spambayes project, which is Copyright 2002-3 # The Python Software Foundation and is covered by the Python Software # Foundation license. __author__ = "Tony Meyer , Tim Stone" __credits__ = "All the Spambayes folk." from __future__ import generators try: True, False except NameError: # Maintain compatibility with Python 2.2 True, False = 1, 0 import socket import os import re import time import sys import getopt import types import email import email.Parser from getpass import getpass from email.Utils import parsedate from spambayes.Options import options from spambayes import tokenizer, storage, message, Dibbler from spambayes.UserInterface import UserInterfaceServer from spambayes.ImapUI import IMAPUserInterface from spambayes.Version import get_version_string from imaplib import IMAP4 from imaplib import Time2Internaldate try: if options["imap", "use_ssl"]: from imaplib import IMAP_SSL as BaseIMAP else: from imaplib import IMAP4 as BaseIMAP except ImportError: from imaplib import IMAP4 as BaseIMAP # global IMAPlib object global imap imap = None # A flag can have any character in the ascii range 32-126 # except for (){ %*"\ FLAG_CHARS = "" for i in range(32, 127): if not chr(i) in ['(', ')', '{', ' ', '%', '*', '"', '\\']: FLAG_CHARS += chr(i) FLAG = r"\\?[" + re.escape(FLAG_CHARS) + r"]+" # The empty flag set "()" doesn't match, so that extract returns # data["FLAGS"] == None FLAGS_RE = re.compile(r"(FLAGS) (\((" + FLAG + r" )*(" + FLAG + r")\))") INTERNALDATE_RE = re.compile(r"(INTERNALDATE) (\"\d{1,2}\-[A-Za-z]{3,3}\-" + r"\d{2,4} \d{2,2}\:\d{2,2}\:\d{2,2} " + r"[\+\-]\d{4,4}\")") RFC822_RE = re.compile(r"(RFC822) (\{[\d]+\})") RFC822_HEADER_RE = re.compile(r"(RFC822.HEADER) (\{[\d]+\})") UID_RE = re.compile(r"(UID) ([\d]+)") FETCH_RESPONSE_RE = re.compile(r"([0-9]+) \(([" + \ re.escape(FLAG_CHARS) + r"\"\{\}\(\)\\ ]*)\)?") LITERAL_RE = re.compile(r"^\{[\d]+\}$") def _extract_fetch_data(response): '''Extract data from the response given to an IMAP FETCH command.''' # Response might be a tuple containing literal data # At the moment, we only handle one literal per response. This # may need to be improved if our code ever asks for something # more complicated (like RFC822.Header and RFC822.Body) if type(response) == types.TupleType: literal = response[1] response = response[0] else: literal = None # the first item will always be the message number mo = FETCH_RESPONSE_RE.match(response) data = {} if mo is None: print """IMAP server gave strange fetch response. Please report this as a bug.""" print response else: data["message_number"] = mo.group(1) response = mo.group(2) # We support the following FETCH items: # FLAGS # INTERNALDATE # RFC822 # UID # RFC822.HEADER # All others are ignored. for r in [FLAGS_RE, INTERNALDATE_RE, RFC822_RE, UID_RE, RFC822_HEADER_RE]: mo = r.search(response) if mo is not None: if LITERAL_RE.match(mo.group(2)): data[mo.group(1)] = literal else: data[mo.group(1)] = mo.group(2) return data class IMAPSession(BaseIMAP): '''A class extending the IMAP4 class, with a few optimizations''' def __init__(self, server, port, debug=0, do_expunge=False): try: BaseIMAP.__init__(self, server, port) except: # A more specific except would be good here, but I get # (in Python 2.2) a generic 'error' and a 'gaierror' # if I pass a valid domain that isn't an IMAP server # or invalid domain (respectively) print "Invalid server or port, please check these settings." sys.exit(-1) self.debug = debug # For efficiency, we remember which folder we are currently # in, and only send a select command to the IMAP server if # we want to *change* folders. This function is used by # both IMAPMessage and IMAPFolder. self.current_folder = None self.do_expunge = do_expunge def login(self, username, pwd): try: BaseIMAP.login(self, username, pwd) # superclass login except BaseIMAP.error, e: if str(e) == "permission denied": print "There was an error logging in to the IMAP server." print "The userid and/or password may be incorrect." sys.exit() else: raise def logout(self): # sign off if self.do_expunge: self.expunge() BaseIMAP.logout(self) # superclass logout def SelectFolder(self, folder): '''A method to point ensuing imap operations at a target folder''' if self.current_folder != folder: if self.current_folder != None: if self.do_expunge: # It is faster to do close() than a single # expunge when we log out (because expunge returns # a list of all the deleted messages, that we don't do # anything with) imap.close() # We *always* use SELECT and not EXAMINE, because this # speeds things up considerably. response = self.select(folder, None) if response[0] != "OK": print "Invalid response to select %s:\n%s" % (folder, response) sys.exit(-1) self.current_folder = folder return response def folder_list(self): '''Return a alphabetical list of all folders available on the server''' response = self.list() if response[0] != "OK": return [] all_folders = response[1] folders = [] for fol in all_folders: # Sigh. Some servers may give us back the folder name as a # literal, so we need to crunch this out. if isinstance(fol, ()): r = re.compile(r"{\d+}") m = r.search(fol[0]) if not m: # Something is wrong here! Skip this folder continue fol = '%s"%s"' % (fol[0][:m.start()], fol[1]) r = re.compile(r"\(([\w\\ ]*)\) ") m = r.search(fol) if not m: # Something is not good with this folder, so skip it. continue name_attributes = fol[:m.end()-1] # IMAP is a truly odd protocol. The delimiter is # only the delimiter for this particular folder - each # folder *may* have a different delimiter self.folder_delimiter = fol[m.end()+1:m.end()+2] # a bit of a hack, but we really need to know if this is # the case if self.folder_delimiter == ',': print """WARNING: Your imap server uses commas as the folder delimiter. This may cause unpredictable errors.""" folders.append(fol[m.end()+5:-1]) folders.sort() return folders def FindMessage(self, id): '''A (potentially very expensive) method to find a message with a given spambayes id (header), and return a message object (no substance).''' # If efficiency becomes a concern, what we could do is store a # dict of key-to-folder, and look in that folder first. (It might # have moved independantly of us, so we would still have to search # if we didn't find it). For the moment, we do an search through # all folders, alphabetically. for folder_name in self.folder_list(): fol = IMAPFolder(folder_name) for msg in fol: if msg.id == id: return msg return None class IMAPMessage(message.SBHeaderMessage): def __init__(self): message.Message.__init__(self) self.folder = None self.previous_folder = None self.rfc822_command = "RFC822.PEEK" self.got_substance = False def setFolder(self, folder): self.folder = folder def _check(self, response, command): if response[0] != "OK": print "Invalid response to %s:\n%s" % (command, response) sys.exit(-1) def extractTime(self): # When we create a new copy of a message, we need to specify # a timestamp for the message. If the message has a valid date # header we use that. Otherwise, we use the current time. message_date = self["Date"] if message_date is not None: parsed_date = parsedate(message_date) if parsed_date is not None: return Time2Internaldate(time.mktime(parsed_date)) else: return Time2Internaldate(time.time()) def get_substance(self): '''Retrieve the RFC822 message from the IMAP server and set as the substance of this message.''' if self.got_substance: return if not self.uid or not self.id: print "Cannot get substance of message without an id and an UID" return imap.SelectFolder(self.folder.name) # We really want to use RFC822.PEEK here, as that doesn't effect # the status of the message. Unfortunately, it appears that not # all IMAP servers support this, even though it is in RFC1730 # Actually, it's not: we should be using BODY.PEEK try: response = imap.uid("FETCH", self.uid, self.rfc822_command) except IMAP4.error: self.rfc822_command = "RFC822" response = imap.uid("FETCH", self.uid, self.rfc822_command) if response[0] != "OK": self.rfc822_command = "RFC822" response = imap.uid("FETCH", self.uid, self.rfc822_command) self._check(response, "uid fetch") data = _extract_fetch_data(response[1][0]) # Annoyingly, we can't just pass over the RFC822 message to an # existing message object (like self) and have it parse it. So # we go through the hoops of creating a new message, and then # copying over all its internals. new_msg = email.Parser.Parser().parsestr(data["RFC822"]) self._headers = new_msg._headers self._unixfrom = new_msg._unixfrom self._payload = new_msg._payload self._charset = new_msg._charset self.preamble = new_msg.preamble self.epilogue = new_msg.epilogue self._default_type = new_msg._default_type if not self.has_key(options["Headers", "mailid_header_name"]): self[options["Headers", "mailid_header_name"]] = self.id self.got_substance = True if options["globals", "verbose"]: sys.stdout.write(chr(8) + "*") def MoveTo(self, dest): '''Note that message should move to another folder. No move is carried out until Save() is called, for efficiency.''' if self.previous_folder is None: self.previous_folder = self.folder self.folder = dest def Save(self): '''Save message to imap server.''' # we can't actually update the message with IMAP # so what we do is create a new message and delete the old one if self.folder is None: raise RuntimeError, """Can't save a message that doesn't have a folder.""" if not self.id: raise RuntimeError, """Can't save a message that doesn't have an id.""" response = imap.uid("FETCH", self.uid, "(FLAGS INTERNALDATE)") self._check(response, 'fetch (flags internaldate)') data = _extract_fetch_data(response[1][0]) if data.has_key("INTERNALDATE"): msg_time = data["INTERNALDATE"] else: msg_time = self.extractTime() if data.has_key("FLAGS"): flags = data["FLAGS"] # The \Recent flag can be fetched, but cannot be stored # We must remove it from the list if it is there. flags = re.sub(r"\\Recent ?|\\ ?Recent", "", flags) else: flags = None response = imap.append(self.folder.name, flags, msg_time, self.as_string()) if response[0] == "NO": # This may be because we have tried to set an invalid flag. # Try again, losing all the flag information, but warn the # user that this has happened. response = imap.append(self.folder.name, None, msg_time, self.as_string()) if response[0] == "OK": print "WARNING: Could not append flags: %s" % (flags,) self._check(response, 'append') if self.previous_folder is None: imap.SelectFolder(self.folder.name) else: imap.SelectFolder(self.previous_folder.name) self.previous_folder = None response = imap.uid("STORE", self.uid, "+FLAGS.SILENT", "(\\Deleted)") self._check(response, 'store') # We need to update the uid, as it will have changed. # Although we don't use the UID to keep track of messages, we do # have to use it for IMAP operations. imap.SelectFolder(self.folder.name) response = imap.uid("SEARCH", "(UNDELETED HEADER " + \ options["Headers", "mailid_header_name"] + \ " " + self.id + ")") self._check(response, 'search') new_id = response[1][0] # Let's hope it doesn't, but, just in case, if the search # turns up empty, we make the assumption that the new # message is the last one with a recent flag if new_id == "": response = imap.uid("SEARCH", "RECENT") new_id = response[1][0] if new_id.find(' ') > -1: ids = new_id.split(' ') new_id = ids[-1] # Ok, now we're in trouble if we still haven't found it. # We make a huge assumption that the new message is the one # with the highest UID (they are sequential, so this will be # ok as long as another message hasn't also arrived) if new_id == "": response = imap.uid("SEARCH", "ALL") new_id = response[1][0] if new_id.find(' ') > -1: ids = new_id.split(' ') new_id = ids[-1] self.uid = new_id # This performs a similar function to email.message_from_string() def imapmessage_from_string(s, _class=IMAPMessage, strict=False): return email.message_from_string(s, _class, strict) class IMAPFolder(object): def __init__(self, folder_name): self.name = folder_name # Unique names for cached messages - see _generate_id below. self.lastBaseMessageName = '' self.uniquifier = 2 def __cmp__(self, obj): '''Two folders are equal if their names are equal''' if obj is None: return False return cmp(self.name, obj.name) def _check(self, response, command): if response[0] != "OK": print "Invalid response to %s:\n%s" % (command, response) sys.exit(-1) def __iter__(self): '''IMAPFolder is iterable''' for key in self.keys(): try: yield self[key] except KeyError: pass def recent_uids(self): '''Returns uids for all the messages in the folder that are flagged as recent, but not flagged as deleted.''' imap.SelectFolder(self.name, True) response = imap.uid("SEARCH", "RECENT UNDELETED") self._check(response, "SEARCH RECENT UNDELETED") return response[1][0].split(' ') def keys(self): '''Returns *uids* for all the messages in the folder not marked as deleted.''' imap.SelectFolder(self.name) response = imap.uid("SEARCH", "UNDELETED") self._check(response, "SEARCH UNDELETED") if response[1][0] == "": return [] return response[1][0].split(' ') def __getitem__(self, key): '''Return message (no substance) matching the given *uid*.''' # We don't retrieve the substances of the message here - you need # to call msg.get_substance() to do that. imap.SelectFolder(self.name) # Using RFC822.HEADER.LINES would be better here, but it seems # that not all servers accept it, even though it is in the RFC response = imap.uid("FETCH", key, "RFC822.HEADER") self._check(response, "uid fetch header") data = _extract_fetch_data(response[1][0]) msg = IMAPMessage() msg.setFolder(self) msg.uid = key r = re.compile(re.escape(options["Headers", "mailid_header_name"]) + \ "\:\s*(\d+(\-\d)?)") mo = r.search(data["RFC822.HEADER"]) if mo is None: msg.setId(self._generate_id()) # Unfortunately, we now have to re-save this message, so that # our id is stored on the IMAP server. Before anyone suggests # it, we can't store it as a flag, because user-defined flags # aren't supported by all IMAP servers. # This will need to be done once per message. msg.get_substance() msg.Save() else: msg.setId(mo.group(1)) if options["globals", "verbose"]: sys.stdout.write(".") return msg # Lifted straight from pop3proxy.py (under the name getNewMessageName) def _generate_id(self): # The message id is the time it arrived, with a uniquifier # appended if two arrive within one clock tick of each other. messageName = "%10.10d" % long(time.time()) if messageName == self.lastBaseMessageName: messageName = "%s-%d" % (messageName, self.uniquifier) self.uniquifier += 1 else: self.lastBaseMessageName = messageName self.uniquifier = 2 return messageName def Train(self, classifier, isSpam): '''Train folder as spam/ham''' num_trained = 0 for msg in self: if msg.GetTrained() == (not isSpam): msg.get_substance() msg.delSBHeaders() classifier.unlearn(msg.asTokens(), not isSpam) # Once the message has been untrained, it's training memory # should reflect that on the off chance that for some reason # the training breaks, which happens on occasion (the # tokenizer is not yet perfect) msg.RememberTrained(None) if msg.GetTrained() is None: msg.get_substance() msg.delSBHeaders() classifier.learn(msg.asTokens(), isSpam) num_trained += 1 msg.RememberTrained(isSpam) if isSpam: move_opt_name = "move_trained_spam_to_folder" else: move_opt_name = "move_trained_ham_to_folder" if options["imap", move_opt_name] != "": msg.MoveTo(IMAPFolder(options["imap", move_opt_name])) msg.Save() return num_trained def Filter(self, classifier, spamfolder, unsurefolder): count = {} count["ham"] = 0 count["spam"] = 0 count["unsure"] = 0 for msg in self: if msg.GetClassification() is None: msg.get_substance() (prob, clues) = classifier.spamprob(msg.asTokens(), evidence=True) # add headers and remember classification msg.addSBHeaders(prob, clues) cls = msg.GetClassification() if cls == options["Hammie", "header_ham_string"]: # we leave ham alone count["ham"] += 1 elif cls == options["Hammie", "header_spam_string"]: msg.MoveTo(spamfolder) count["spam"] += 1 else: msg.MoveTo(unsurefolder) count["unsure"] += 1 msg.Save() return count class IMAPFilter(object): def __init__(self, classifier): self.spam_folder = IMAPFolder(options["imap", "spam_folder"]) self.unsure_folder = IMAPFolder(options["imap", "unsure_folder"]) self.classifier = classifier def Train(self): if options["globals", "verbose"]: t = time.time() total_ham_trained = 0 total_spam_trained = 0 if options["imap", "ham_train_folders"] != "": ham_training_folders = options["imap", "ham_train_folders"] for fol in ham_training_folders: # Select the folder to make sure it exists imap.SelectFolder(fol) if options['globals', 'verbose']: print " Training ham folder %s" % (fol) folder = IMAPFolder(fol) num_ham_trained = folder.Train(self.classifier, False) total_ham_trained += num_ham_trained if options['globals', 'verbose']: print " %s trained." % (num_ham_trained) if options["imap", "spam_train_folders"] != "": spam_training_folders = options["imap", "spam_train_folders"] for fol in spam_training_folders: # Select the folder to make sure it exists imap.SelectFolder(fol) if options['globals', 'verbose']: print " Training spam folder %s" % (fol) folder = IMAPFolder(fol) num_spam_trained = folder.Train(self.classifier, True) total_spam_trained += num_spam_trained if options['globals', 'verbose']: print " %s trained." % (num_spam_trained) if total_ham_trained or total_spam_trained: self.classifier.store() if options["globals", "verbose"]: print "Training took %s seconds, %s messages were trained" \ % (time.time() - t, total_ham_trained + total_spam_trained) def Filter(self): if options["globals", "verbose"]: t = time.time() count = None # Select the spam folder and unsure folder to make sure they exist imap.SelectFolder(self.spam_folder.name) imap.SelectFolder(self.unsure_folder.name) for filter_folder in options["imap", "filter_folders"]: # Select the folder to make sure it exists imap.SelectFolder(filter_folder) folder = IMAPFolder(filter_folder) count = folder.Filter(self.classifier, self.spam_folder, self.unsure_folder) if options["globals", "verbose"]: if count is not None: print "\nClassified %s ham, %s spam, and %s unsure." % \ (count["ham"], count["spam"], count["unsure"]) print "Classifying took", time.time() - t, "seconds." def run(): global imap try: opts, args = getopt.getopt(sys.argv[1:], 'hbtcvpl:e:i:d:D:') except getopt.error, msg: print >>sys.stderr, str(msg) + '\n\n' + __doc__ sys.exit() bdbname = options["Storage", "persistent_storage_file"] useDBM = options["Storage", "persistent_use_database"] doTrain = False doClassify = False doExpunge = options["imap", "expunge"] imapDebug = 0 sleepTime = 0 promptForPass = False launchUI = False server = "" username = "" for opt, arg in opts: if opt == '-h': print >>sys.stderr, __doc__ sys.exit() elif opt == '-d': useDBM = False bdbname = arg elif opt == '-D': useDBM = True bdbname = arg elif opt == "-b": launchUI = True elif opt == '-t': doTrain = True elif opt == '-p': promptForPass = True elif opt == '-c': doClassify = True elif opt == '-v': options["globals", "verbose"] = True elif opt == '-e': if arg == 'y': doExpunge = True else: doExpunge = False elif opt == '-i': imapDebug = int(arg) elif opt == '-l': sleepTime = int(arg) * 60 # Let the user know what they are using... print get_version_string("IMAP Filter") print "and engine %s.\n" % (get_version_string(),) if not (doClassify or doTrain or launchUI): print "-b, -c, or -t operands must be specified." print "Please use the -h operand for help." sys.exit() if (launchUI and (doClassify or doTrain)): print """ -b option is exclusive with -c and -t options. The user interface will be launched, but no classification or training will be performed.""" bdbname = os.path.expanduser(bdbname) if options["globals", "verbose"]: print "Loading database %s..." % (bdbname), classifier = storage.open_storage(bdbname, useDBM) if options["globals", "verbose"]: print "Done." if options["imap", "server"]: # The options class is ahead of us here: # it knows that imap:server will eventually be able to have # multiple values, but for the moment, we just use the first one server = options["imap", "server"] if len(server) > 0: server = server[0] username = options["imap", "username"] if len(username) > 0: username = username[0] if not promptForPass: pwd = options["imap", "password"] if len(pwd) > 0: pwd = pwd[0] else: pwd = None if not launchUI: print "You need to specify both a server and a username." sys.exit() if promptForPass: pwd = getpass() if server.find(':') > -1: server, port = server.split(':', 1) port = int(port) else: if options["imap", "use_ssl"]: port = 993 else: port = 143 imap_filter = IMAPFilter(classifier) # Web interface if launchUI: if server != "": imap = IMAPSession(server, port, imapDebug, doExpunge) httpServer = UserInterfaceServer(options["html_ui", "port"]) httpServer.register(IMAPUserInterface(classifier, imap, pwd)) Dibbler.run(launchBrowser=launchUI) else: while True: imap = IMAPSession(server, port, imapDebug, doExpunge) imap.login(username, pwd) if doTrain: if options["globals", "verbose"]: print "Training" imap_filter.Train() if doClassify: if options["globals", "verbose"]: print "Classifying" imap_filter.Filter() imap.logout() if sleepTime: time.sleep(sleepTime) else: break if __name__ == '__main__': run() --- NEW FILE: sb_mailsort.py --- #! /usr/bin/env python """\ To train: %(program)s -t ham.mbox spam.mbox To filter mail (using .forward or .qmail): |%(program)s Maildir/ Mail/Spam/ To print the score and top evidence for a message or messages: %(program)s -s message [message ...] """ SPAM_CUTOFF = 0.57 SIZE_LIMIT = 5000000 # messages larger are not analyzed BLOCK_SIZE = 10000 RC_DIR = "~/.spambayes" DB_FILE = RC_DIR + "/wordprobs.cdb" CONFIG_FILE = RC_DIR + "/bayescustomize.ini" import sys import os import getopt import email import time import signal import socket import email DB_FILE = os.path.expanduser(DB_FILE) def import_spambayes(): global mboxutils, CdbClassifier, tokenize if not os.environ.has_key('BAYESCUSTOMIZE'): os.environ['BAYESCUSTOMIZE'] = os.path.expanduser(CONFIG_FILE) from spambayes import mboxutils from spambayes.cdb_classifier import CdbClassifier from spambayes.tokenizer import tokenize try: True, False except NameError: # Maintain compatibility with Python 2.2 True, False = 1, 0 program = sys.argv[0] # For usage(); referenced by docstring above def usage(code, msg=''): """Print usage message and sys.exit(code).""" if msg: print >> sys.stderr, msg print >> sys.stderr print >> sys.stderr, __doc__ % globals() sys.exit(code) def maketmp(dir): hostname = socket.gethostname() pid = os.getpid() fd = -1 for x in xrange(200): filename = "%d.%d.%s" % (time.time(), pid, hostname) pathname = "%s/tmp/%s" % (dir, filename) try: fd = os.open(pathname, os.O_WRONLY|os.O_CREAT|os.O_EXCL, 0600) except IOError, exc: if exc[i] not in (errno.EINT, errno.EEXIST): raise else: break time.sleep(2) if fd == -1: raise SystemExit, "could not create a mail file" return (os.fdopen(fd, "wb"), pathname, filename) def train(bayes, msgs, is_spam): """Train bayes with all messages from a mailbox.""" mbox = mboxutils.getmbox(msgs) for msg in mbox: bayes.learn(tokenize(msg), is_spam) def train_messages(ham_name, spam_name): """Create database using messages.""" rc_dir = os.path.expanduser(RC_DIR) if not os.path.exists(rc_dir): print "Creating", RC_DIR, "directory..." os.mkdir(rc_dir) bayes = CdbClassifier() print 'Training with ham...' train(bayes, ham_name, False) print 'Training with spam...' train(bayes, spam_name, True) print 'Update probabilities and writing DB...' db = open(DB_FILE, "wb") bayes.save_wordinfo(db) db.close() print 'done' def filter_message(hamdir, spamdir): signal.signal(signal.SIGALRM, lambda s: sys.exit(1)) signal.alarm(24 * 60 * 60) # write message to temporary file (must be on same partition) tmpfile, pathname, filename = maketmp(hamdir) try: tmpfile.write(os.environ.get("DTLINE", "")) # delivered-to line bytes = 0 blocks = [] while 1: block = sys.stdin.read(BLOCK_SIZE) if not block: break bytes += len(block) if bytes < SIZE_LIMIT: blocks.append(block) tmpfile.write(block) tmpfile.close() if bytes < SIZE_LIMIT: msgdata = ''.join(blocks) del blocks msg = email.message_from_string(msgdata) del msgdata bayes = CdbClassifier(open(DB_FILE, 'rb')) prob = bayes.spamprob(tokenize(msg)) else: prob = 0.0 if prob > SPAM_CUTOFF: os.rename(pathname, "%s/new/%s" % (spamdir, filename)) else: os.rename(pathname, "%s/new/%s" % (hamdir, filename)) except: os.unlink(pathname) raise def print_message_score(msg_name, msg_fp): msg = email.message_from_file(msg_fp) bayes = CdbClassifier(open(DB_FILE, 'rb')) prob, evidence = bayes.spamprob(tokenize(msg), evidence=True) print msg_name, prob for word, prob in evidence: print ' ', `word`, prob def main(): global DB_FILE, CONFIG_FILE try: opts, args = getopt.getopt(sys.argv[1:], 'tsd:c:') except getopt.error, msg: usage(2, msg) mode = 'sort' for opt, val in opts: if opt == '-t': mode = 'train' elif opt == '-s': mode = 'score' elif opt == '-d': DB_FILE = val elif opt == '-c': CONFIG_FILE = val else: assert 0, 'invalid option' import_spambayes() if mode == 'sort': if len(args) != 2: usage(2, 'wrong number of arguments') filter_message(args[0], args[1]) elif mode == 'train': if len(args) != 2: usage(2, 'wrong number of arguments') train_messages(args[0], args[1]) elif mode == 'score': if args: for msg in args: print_message_score(msg, open(msg)) else: print_message_score('', sys.stdin) if __name__ == "__main__": main() --- NEW FILE: sb_mboxtrain.py --- #! /usr/bin/env python ### Train spambayes on all previously-untrained messages in a mailbox. ### ### This keeps track of messages it's already trained by adding an ### X-Spambayes-Trained: header to each one. Then, if you move one to ### another folder, it will retrain that message. You would want to run ### this from a cron job on your server. """Usage: %(program)s [OPTIONS] ... Where OPTIONS is one or more of: -h show usage and exit -d DBNAME use the DBM store. A DBM file is larger than the pickle and creating it is slower, but loading it is much faster, especially for large word databases. Recommended for use with hammiefilter or any procmail-based filter. -D DBNAME use the pickle store. A pickle is smaller and faster to create, but much slower to load. Recommended for use with pop3proxy and hammiesrv. -g PATH mbox or directory of known good messages (non-spam) to train on. Can be specified more than once. -s PATH mbox or directory of known spam messages to train on. Can be specified more than once. -f force training, ignoring the trained header. Use this if you need to rebuild your database from scratch. -q quiet mode; no output -n train mail residing in "new" directory, in addition to "cur" directory, which is always trained (Maildir only) -r remove mail which was trained on (Maildir only) """ try: True, False except NameError: # Maintain compatibility with Python 2.2 True, False = 1, 0 import sys, os, getopt from spambayes import hammie, mboxutils from spambayes.Options import options program = sys.argv[0] TRAINED_HDR = "X-Spambayes-Trained" loud = True def msg_train(h, msg, is_spam, force): """Train bayes with a single message.""" # XXX: big hack -- why is email.Message unable to represent # multipart/alternative? try: msg.as_string() except TypeError: # We'll be unable to represent this as text :( return False if is_spam: spamtxt = options["Headers", "header_spam_string"] else: spamtxt = options["Headers", "header_ham_string"] oldtxt = msg.get(TRAINED_HDR) if force: # Train no matter what. if oldtxt != None: del msg[TRAINED_HDR] elif oldtxt == spamtxt: # Skip this one, we've already trained with it. return False elif oldtxt != None: # It's been trained, but as something else. Untrain. del msg[TRAINED_HDR] h.untrain(msg, not is_spam) h.train(msg, is_spam) msg.add_header(TRAINED_HDR, spamtxt) return True def maildir_train(h, path, is_spam, force, removetrained): """Train bayes with all messages from a maildir.""" if loud: print " Reading %s as Maildir" % (path,) import time import socket pid = os.getpid() host = socket.gethostname() counter = 0 trained = 0 for fn in os.listdir(path): cfn = os.path.join(path, fn) tfn = os.path.normpath(os.path.join(path, "..", "tmp", "%d.%d_%d.%s" % (time.time(), pid, counter, host))) if (os.path.isdir(cfn)): continue counter += 1 if loud: sys.stdout.write(" %s \r" % fn) sys.stdout.flush() f = file(cfn, "rb") msg = mboxutils.get_message(f) f.close() if not msg_train(h, msg, is_spam, force): continue trained += 1 f = file(tfn, "wb") f.write(msg.as_string()) f.close() # XXX: This will raise an exception on Windows. Do any Windows # people actually use Maildirs? os.rename(tfn, cfn) if (removetrained): os.unlink(cfn) if loud: print (" Trained %d out of %d messages " % (trained, counter)) def mbox_train(h, path, is_spam, force): """Train bayes with a Unix mbox""" if loud: print " Reading as Unix mbox" import mailbox import fcntl import tempfile # Open and lock the mailbox. Some systems require it be opened for # writes in order to assert an exclusive lock. f = file(path, "r+b") fcntl.flock(f, fcntl.LOCK_EX) mbox = mailbox.PortableUnixMailbox(f, mboxutils.get_message) outf = os.tmpfile() counter = 0 trained = 0 for msg in mbox: counter += 1 if loud: sys.stdout.write(" %s\r" % counter) sys.stdout.flush() if msg_train(h, msg, is_spam, force): trained += 1 # Write it out with the Unix "From " line outf.write(msg.as_string(True)) outf.seek(0) try: os.ftruncate(f.fileno(), 0) f.seek(0) except: # If anything goes wrong, don't try to write print "Problem truncating mbox--nothing written" raise try: for line in outf.xreadlines(): f.write(line) except: print >> sys.stderr ("Problem writing mbox! Sorry, " "I tried my best, but your mail " "may be corrupted.") raise fcntl.lockf(f, fcntl.LOCK_UN) f.close() if loud: print (" Trained %d out of %d messages " % (trained, counter)) def mhdir_train(h, path, is_spam, force): """Train bayes with an mh directory""" if loud: print " Reading as MH mailbox" import glob counter = 0 trained = 0 for fn in glob.glob(os.path.join(path, "[0-9]*")): counter += 1 cfn = fn tfn = os.path.join(path, "spambayes.tmp") if loud: sys.stdout.write(" %s \r" % fn) sys.stdout.flush() f = file(fn, "rb") msg = mboxutils.get_message(f) f.close() msg_train(h, msg, is_spam, force) trained += 1 f = file(tfn, "wb") f.write(msg.as_string()) f.close() # XXX: This will raise an exception on Windows. Do any Windows # people actually use MH directories? os.rename(tfn, cfn) if loud: print (" Trained %d out of %d messages " % (trained, counter)) def train(h, path, is_spam, force, trainnew, removetrained): if not os.path.exists(path): raise ValueError("Nonexistent path: %s" % path) elif os.path.isfile(path): mbox_train(h, path, is_spam, force) elif os.path.isdir(os.path.join(path, "cur")): maildir_train(h, os.path.join(path, "cur"), is_spam, force, removetrained) if trainnew: maildir_train(h, os.path.join(path, "new"), is_spam, force, removetrained) elif os.path.isdir(path): mhdir_train(h, path, is_spam, force) else: raise ValueError("Unable to determine mailbox type: " + path) def usage(code, msg=''): """Print usage message and sys.exit(code).""" if msg: print >> sys.stderr, msg print >> sys.stderr print >> sys.stderr, __doc__ % globals() sys.exit(code) def main(): """Main program; parse options and go.""" global loud try: opts, args = getopt.getopt(sys.argv[1:], 'hfqnrd:D:g:s:') except getopt.error, msg: usage(2, msg) if not opts: usage(2, "No options given") pck = None usedb = None force = False trainnew = False removetrained = False good = [] spam = [] for opt, arg in opts: if opt == '-h': usage(0) elif opt == "-f": force = True elif opt == "-n": trainnew = True elif opt == "-q": loud = False elif opt == '-g': good.append(arg) elif opt == '-s': spam.append(arg) elif opt == "-r": removetrained = True elif opt == "-d": usedb = True pck = arg elif opt == "-D": usedb = False pck = arg if args: usage(2, "Positional arguments not allowed") if usedb == None: usage(2, "Must specify one of -d or -D") h = hammie.open(pck, usedb, "c") for g in good: if loud: print "Training ham (%s):" % g train(h, g, False, force, trainnew, removetrained) save = True for s in spam: if loud: print "Training spam (%s):" % s train(h, s, True, force, trainnew, removetrained) save = True if save: h.store() if __name__ == "__main__": main() --- NEW FILE: sb_notesfilter.py --- #! /usr/bin/env python '''notesfilter.py - Lotus Notes Spambayes interface. Classes: Abstract: This module uses Spambayes as a filter against a Lotus Notes mail database. The Notes client must be running when this process is executed. It requires a Notes folder, named as a parameter, with four subfolders: Spam Ham Train as Spam Train as Ham Depending on the execution parameters, it will do any or all of the following steps, in the order given. 1. Train Spam from the Train as Spam folder (-t option) 2. Train Ham from the Train as Ham folder (-t option) 3. Replicate (-r option) 4. Classify the inbox (-c option) Mail that is to be trained as spam should be manually moved to that folder by the user. Likewise mail that is to be trained as ham. After training, spam is moved to the Spam folder and ham is moved to the Ham folder. Replication takes place if a remote server has been specified. This step may take a long time, depending on replication parameters and how much information there is to download, as well as line speed and server load. Please be patient if you run with replication. There is currently no progress bar or anything like that to tell you that it's working, but it is and will complete eventually. There is also no mechanism for notifying you that the replication failed. If it did, there is no harm done, and the program will continue execution. Mail that is classified as Spam is moved from the inbox to the Train as Spam folder. You should occasionally review your Spam folder for Ham that has mistakenly been classified as Spam. If there is any there, move it to the Train as Ham folder, so Spambayes will be less likely to make this mistake again. Mail that is classified as Ham or Unsure is left in the inbox. There is currently no means of telling if a mail was classified as Ham or Unsure. You should occasionally select some Ham and move it to the Train as Ham folder, so Spambayes can tell the difference between Spam and Ham. The goal is to maintain a relative balance between the number of Spam and the number of Ham that have been trained into the database. These numbers are reported every time this program executes. However, if the amount of Spam you receive far exceeds the amount of Ham you receive, it may be very difficult to maintain this balance. This is not a matter of great concern. Spambayes will still make very few mistakes in this circumstance. But, if this is the case, you should review your Spam folder for falsely classified Ham, and retrain those that you find, on a regular basis. This will prevent statistical error accumulation, which if allowed to continue, would cause Spambayes to tend to classify everything as Spam. Because there is no programmatic way to determine if a particular mail has been previously processed by this classification program, it keeps a pickled dictionary of notes mail ids, so that once a mail has been classified, it will not be classified again. The non-existence of is index file, named .sbindex, indicates to the system that this is an initialization execution. Rather than classify the inbox in this case, the contents of the inbox are placed in the index to note the 'starting point' of the system. After that, any new messages in the inbox are eligible for classification. Usage: notesfilter [options] note: option values with spaces in them must be enclosed in double quotes options: -d dbname : pickled training database filename -D dbname : dbm training database filename -l dbname : database filename of local mail replica e.g. localmail.nsf -r server : server address of the server mail database e.g. d27ml602/27/M/IBM if specified, will initiate a replication -f folder : Name of spambayes folder must have subfolders: Spam Ham Train as Spam Train as Ham -t : train contents of Train as Spam and Train as Ham -c : classify inbox -h : help -p : prompt "Press Enter to end" before ending This is useful for automated executions where the statistics output would otherwise be lost when the window closes. Examples: Replicate and classify inbox notesfilter -c -d notesbayes -r mynoteserv -l mail.nsf -f Spambayes Train Spam and Ham, then classify inbox notesfilter -t -c -d notesbayes -l mail.nsf -f Spambayes Replicate, then classify inbox notesfilter -c -d test7 -l mail.nsf -r nynoteserv -f Spambayes To Do: o Dump/purge notesindex file o Create correct folders if they do not exist o Options for some of this stuff? o pop3proxy style training/configuration interface? o parameter to retrain? o Suggestions? ''' # This module is part of the spambayes project, which is Copyright 2002 # The Python Software Foundation and is covered by the Python Software # Foundation license. __author__ = "Tim Stone " __credits__ = "Mark Hammond, for his remarkable win32 modules." from __future__ import generators try: True, False except NameError: # Maintain compatibility with Python 2.2 True, False = 1, 0 def bool(val): return not not val import sys from spambayes import tokenizer, storage from spambayes.Options import options import cPickle as pickle import errno import win32com.client import pywintypes import getopt def classifyInbox(v, vmoveto, bayes, ldbname, notesindex): # the notesindex hash ensures that a message is looked at only once if len(notesindex.keys()) == 0: firsttime = 1 else: firsttime = 0 docstomove = [] numham = 0 numspam = 0 numuns = 0 numdocs = 0 doc = v.GetFirstDocument() while doc: nid = doc.NOTEID if firsttime: notesindex[nid] = 'never classified' else: if not notesindex.has_key(nid): numdocs += 1 # Notes returns strings in unicode, and the Python # uni-decoder has trouble with these strings when # you try to print them. So don't... # The com interface returns basic data types as tuples # only, thus the subscript on GetItemValue try: subj = doc.GetItemValue('Subject')[0] except: subj = 'No Subject' try: body = doc.GetItemValue('Body')[0] except: body = 'No Body' message = "Subject: %s\r\n\r\n%s" % (subj, body) # generate_long_skips = True blows up on occasion, # probably due to this unicode problem. options["Tokenizer", "generate_long_skips"] = False tokens = tokenizer.tokenize(message) prob, clues = bayes.spamprob(tokens, evidence=True) if prob < options["Categorization", "ham_cutoff"]: disposition = options["Hammie", "header_ham_string"] numham += 1 elif prob > options["Categorization", "spam_cutoff"]: disposition = options["Hammie", "header_spam_string"] docstomove += [doc] numspam += 1 else: disposition = options["Hammie", "header_unsure_string"] numuns += 1 notesindex[nid] = 'classified' try: print "%s spamprob is %s" % (subj[:30], prob) except UnicodeError: print " spamprob is %s" % (prob) doc = v.GetNextDocument(doc) # docstomove list is built because moving documents in the middle of # the classification loop looses the iterator position for doc in docstomove: doc.RemoveFromFolder(v.Name) doc.PutInFolder(vmoveto.Name) print "%s documents processed" % (numdocs) print " %s classified as spam" % (numspam) print " %s classified as ham" % (numham) print " %s classified as unsure" % (numuns) def processAndTrain(v, vmoveto, bayes, is_spam, notesindex): if is_spam: str = options["Hammie", "header_spam_string"] else: str = options["Hammie", "header_ham_string"] print "Training %s" % (str) docstomove = [] doc = v.GetFirstDocument() while doc: try: subj = doc.GetItemValue('Subject')[0] except: subj = 'No Subject' try: body = doc.GetItemValue('Body')[0] except: body = 'No Body' message = "Subject: %s\r\n%s" % (subj, body) options["Tokenizer", "generate_long_skips"] = False tokens = tokenizer.tokenize(message) nid = doc.NOTEID if notesindex.has_key(nid): trainedas = notesindex[nid] if trainedas == options["Hammie", "header_spam_string"] and \ not is_spam: # msg is trained as spam, is to be retrained as ham bayes.unlearn(tokens, True) elif trainedas == options["Hammie", "header_ham_string"] and \ is_spam: # msg is trained as ham, is to be retrained as spam bayes.unlearn(tokens, False) bayes.learn(tokens, is_spam) notesindex[nid] = str docstomove += [doc] doc = v.GetNextDocument(doc) for doc in docstomove: doc.RemoveFromFolder(v.Name) doc.PutInFolder(vmoveto.Name) print "%s documents trained" % (len(docstomove)) def run(bdbname, useDBM, ldbname, rdbname, foldname, doTrain, doClassify): if useDBM: bayes = storage.DBDictClassifier(bdbname) else: bayes = storage.PickledClassifier(bdbname) try: fp = open("%s.sbindex" % (ldbname), 'rb') except IOError, e: if e.errno != errno.ENOENT: raise notesindex = {} print "%s.sbindex file not found, this is a first time run" \ % (ldbname) print "No classification will be performed" else: notesindex = pickle.load(fp) fp.close() sess = win32com.client.Dispatch("Lotus.NotesSession") try: sess.initialize() except pywintypes.com_error: print "Session aborted" sys.exit() db = sess.GetDatabase("",ldbname) vinbox = db.getView('($Inbox)') vspam = db.getView("%s\Spam" % (foldname)) vham = db.getView("%s\Ham" % (foldname)) vtrainspam = db.getView("%s\Train as Spam" % (foldname)) vtrainham = db.getView("%s\Train as Ham" % (foldname)) if doTrain: processAndTrain(vtrainspam, vspam, bayes, True, notesindex) # for some reason, using inbox as a target here loses the mail processAndTrain(vtrainham, vham, bayes, False, notesindex) if rdbname: print "Replicating..." db.Replicate(rdbname) print "Done" if doClassify: classifyInbox(vinbox, vtrainspam, bayes, ldbname, notesindex) print "The Spambayes database currently has %s Spam and %s Ham" \ % (bayes.nspam, bayes.nham) bayes.store() fp = open("%s.sbindex" % (ldbname), 'wb') pickle.dump(notesindex, fp) fp.close() if __name__ == '__main__': try: opts, args = getopt.getopt(sys.argv[1:], 'htcpd:D:l:r:f:') except getopt.error, msg: print >>sys.stderr, str(msg) + '\n\n' + __doc__ sys.exit() bdbname = None # bayes database name ldbname = None # local notes database name rdbname = None # remote notes database location sbfname = None # spambayes folder name doTrain = False doClassify = False doPrompt = False for opt, arg in opts: if opt == '-h': print >>sys.stderr, __doc__ sys.exit() elif opt == '-d': useDBM = False bdbname = arg elif opt == '-D': useDBM = True bdbname = arg elif opt == '-l': ldbname = arg elif opt == '-r': rdbname = arg elif opt == '-f': sbfname = arg elif opt == '-t': doTrain = True elif opt == '-c': doClassify = True elif opt == '-p': doPrompt = True if (bdbname and ldbname and sbfname and (doTrain or doClassify)): run(bdbname, useDBM, ldbname, rdbname, \ sbfname, doTrain, doClassify) if doPrompt: try: key = input("Press Enter to end") except SyntaxError: pass else: print >>sys.stderr, __doc__ --- NEW FILE: sb_pop3dnd.py --- #!/usr/bin/env python from __future__ import generators """ Overkill (someone *please* come up with something to call this script!) This application is a twisted cross between a POP3 proxy and an IMAP server. It sits between your mail client and your POP3 server (like any other POP3 proxy). While messages classified as ham are simply passed through the proxy, messages that are classified as spam or unsure are intercepted and passed to the IMAP server. The IMAP server offers three folders - one where messages classified as spam end up, one for messages it is unsure about, and one for training ham. In other words, to use this application, setup your mail client to connect to localhost, rather than directly to your POP3 server. Additionally, add a new IMAP account, also connecting to localhost. Setup the application via the web interface, and you are ready to go. Good messages will appear as per normal, but you will also have two new incoming folders, one for spam and one for ham. To train SpamBayes, use the spam folder, and the 'train_as_ham' folder. Any messages in these folders will be trained appropriately. This means that all messages that SpamBayes classifies as spam will also be trained as such. If you receive any 'false positives' (ham classified as spam), you *must* copy the message into the 'train_as_ham' folder to correct the training. You may also place any saved spam messages you have into this folder. So that SpamBayes knows about ham as well as spam, you will also need to move or copy mail into the 'train_as_ham' folder. These may come from the unsure folder, or from any other mail you have saved. It is a good idea to leave messages in the 'train_as_ham' and 'spam' folders, so that you can retrain from scratch if required. (However, you should always clear out your unsure folder, preferably moving or copying the messages into the appropriate training folder). This SpamBayes application is designed to work with Outlook Express, and provide the same sort of ease of use as the Outlook plugin. Although the majority of development and testing has been done with Outlook Express, any mail client that supports both IMAP and POP3 should be able to use this application - if the client enables the user to work with an IMAP account and POP3 account side-by-side (and move messages between them), then it should work equally as well as Outlook Express. This module includes the following classes: o IMAPFileMessage o IMAPFileMessageFactory o IMAPMailbox o SpambayesMailbox o Trainer o SpambayesAccount o SpambayesIMAPServer o OneParameterFactory o MyBayesProxy o MyBayesProxyListener o IMAPState """ todo = """ o Message flags are currently not persisted, but should be. The IMAPFileMessage class should be extended to do this. The same goes for the 'internaldate' of the message. o The RECENT flag should be unset at some point, but when? The RFC says that a message is recent if this is the first session to be notified about the message. Perhaps this can be done simply by *not* persisting this flag - i.e. the flag is always loaded as not recent, and only new messages are recent. The RFC says that if it is not possible to determine, then all messages should be recent, and this is what we currently do. o The Mailbox should be calling the appropriate listener functions (currently only newMessages is called on addMessage). flagsChanged should also be called on store, addMessage, or ??? o We cannot currently get part of a message via the BODY calls (with the <> operands), or get a part of a MIME message (by prepending a number). This should be added! o If the user clicks the 'save and shutdown' button on the web interface, this will only kill the POP3 proxy and web interface threads, and not the IMAP server. We need to monitor the thread that we kick off, and if it dies, we should die too. Need to figure out how to do this in twisted. o Suggestions? """ # This module is part of the spambayes project, which is Copyright 2002-3 # The Python Software Foundation and is covered by the Python Software # Foundation license. __author__ = "Tony Meyer " __credits__ = "All the Spambayes folk." try: True, False except NameError: # Maintain compatibility with Python 2.2 True, False = 1, 0 import os import re import sys import md5 import time import errno import types import thread import getopt import imaplib import operator import StringIO import email.Utils from twisted import cred from twisted.internet import defer from twisted.internet import reactor from twisted.internet.app import Application from twisted.internet.defer import maybeDeferred from twisted.internet.protocol import ServerFactory from twisted.protocols.imap4 import parseNestedParens, parseIdList from twisted.protocols.imap4 import IllegalClientResponse, IAccount from twisted.protocols.imap4 import collapseNestedLists, MessageSet from twisted.protocols.imap4 import IMAP4Server, MemoryAccount, IMailbox from twisted.protocols.imap4 import IMailboxListener, collapseNestedLists # Provide for those that don't have spambayes on their PYTHONPATH sys.path.insert(-1, os.path.dirname(os.getcwd())) from spambayes.Options import options from spambayes.message import Message from spambayes.tokenizer import tokenize from spambayes import FileCorpus, Dibbler from spambayes.Version import get_version_string from spambayes.ServerUI import ServerUserInterface from spambayes.UserInterface import UserInterfaceServer from pop3proxy import POP3ProxyBase, State, _addressPortStr, _recreateState def ensureDir(dirname): """Ensure that the given directory exists - in other words, if it does not exist, attempt to create it.""" try: os.mkdir(dirname) if options["globals", "verbose"]: print "Creating directory", dirname except OSError, e: if e.errno != errno.EEXIST: raise class IMAPFileMessage(FileCorpus.FileMessage): '''IMAP Message that persists as a file system artifact.''' def __init__(self, file_name, directory): """Constructor(message file name, corpus directory name).""" FileCorpus.FileMessage.__init__(self, file_name, directory) self.id = file_name self.directory = directory self.date = imaplib.Time2Internaldate(time.time())[1:-1] self.clear_flags() # IMessage implementation def getHeaders(self, negate, names): """Retrieve a group of message headers.""" headers = {} if not isinstance(names, tuple): names = (names,) for header, value in self.items(): if (header.upper() in names and not negate) or names == (): headers[header.upper()] = value return headers def getFlags(self): """Retrieve the flags associated with this message.""" return self._flags_iter() def _flags_iter(self): if self.deleted: yield "\\DELETED" if self.answered: yield "\\ANSWERED" if self.flagged: yield "\\FLAGGED" if self.seen: yield "\\SEEN" if self.draft: yield "\\DRAFT" if self.draft: yield "\\RECENT" def getInternalDate(self): """Retrieve the date internally associated with this message.""" return self.date def getBodyFile(self): """Retrieve a file object containing the body of this message.""" # Note only body, not headers! s = StringIO.StringIO() s.write(self.body()) s.seek(0) return s #return file(os.path.join(self.directory, self.id), "r") def getSize(self): """Retrieve the total size, in octets, of this message.""" return len(self.as_string()) def getUID(self): """Retrieve the unique identifier associated with this message.""" return self.id def getSubPart(self, part): """Retrieve a MIME sub-message @type part: C{int} @param part: The number of the part to retrieve, indexed from 0. @rtype: Any object implementing C{IMessage}. @return: The specified sub-part. """ # IMessage implementation ends def clear_flags(self): """Set all message flags to false.""" self.deleted = False self.answered = False self.flagged = False self.seen = False self.draft = False self.recent = False def set_flag(self, flag, value): # invalid flags are ignored flag = flag.upper() if flag == "\\DELETED": self.deleted = value elif flag == "\\ANSWERED": self.answered = value elif flag == "\\FLAGGED": self.flagged = value elif flag == "\\SEEN": self.seen = value elif flag == "\\DRAFT": self.draft = value else: print "Tried to set invalid flag", flag, "to", value def flags(self): """Return the message flags.""" all_flags = [] if self.deleted: all_flags.append("\\DELETED") if self.answered: all_flags.append("\\ANSWERED") if self.flagged: all_flags.append("\\FLAGGED") if self.seen: all_flags.append("\\SEEN") if self.draft: all_flags.append("\\DRAFT") if self.draft: all_flags.append("\\RECENT") return all_flags def train(self, classifier, isSpam): if self.GetTrained() == (not isSpam): classifier.unlearn(self.asTokens(), not isSpam) self.RememberTrained(None) if self.GetTrained() is None: classifier.learn(self.asTokens(), isSpam) self.RememberTrained(isSpam) classifier.store() def structure(self, ext=False): """Body structure data describes the MIME-IMB format of a message and consists of a sequence of mime type, mime subtype, parameters, content id, description, encoding, and size. The fields following the size field are variable: if the mime type/subtype is message/rfc822, the contained message's envelope information, body structure data, and number of lines of text; if the mime type is text, the number of lines of text. Extension fields may also be included; if present, they are: the MD5 hash of the body, body disposition, body language.""" s = [] for part in self.walk(): if part.get_content_charset() is not None: charset = ("charset", part.get_content_charset()) else: charset = None part_s = [part.get_main_type(), part.get_subtype(), charset, part.get('Content-Id'), part.get('Content-Description'), part.get('Content-Transfer-Encoding'), str(len(part.as_string()))] #if part.get_type() == "message/rfc822": # part_s.extend([envelope, body_structure_data, # part.as_string().count("\n")]) #elif part.get_main_type() == "text": if part.get_main_type() == "text": part_s.append(str(part.as_string().count("\n"))) if ext: part_s.extend([md5.new(part.as_string()).digest(), part.get('Content-Disposition'), part.get('Content-Language')]) s.append(part_s) if len(s) == 1: return s[0] return s def body(self): rfc822 = self.as_string() bodyRE = re.compile(r"\r?\n(\r?\n)(.*)", re.DOTALL + re.MULTILINE) bmatch = bodyRE.search(rfc822) return bmatch.group(2) def headers(self): rfc822 = self.as_string() bodyRE = re.compile(r"\r?\n(\r?\n)(.*)", re.DOTALL + re.MULTILINE) bmatch = bodyRE.search(rfc822) return rfc822[:bmatch.start(2)] def on(self, date1, date2): "contained within the date" raise NotImplementedError def before(self, date1, date2): "before the date" raise NotImplementedError def since(self, date1, date2): "within or after the date" raise NotImplementedError def string_contains(self, whole, sub): return whole.find(sub) != -1 def matches(self, criteria): """Return True iff the messages matches the specified IMAP criteria.""" match_tests = {"ALL" : [(True, True)], "ANSWERED" : [(self.answered, True)], "DELETED" : [(self.deleted, True)], "DRAFT" : [(self.draft, True)], "FLAGGED" : [(self.flagged, True)], "NEW" : [(self.recent, True), (self.seen, False)], "RECENT" : [(self.recent, True)], "SEEN" : [(self.seen, True)], "UNANSWERED" : [(self.answered, False)], "UNDELETED" : [(self.deleted, False)], "UNDRAFT" : [(self.draft, False)], "UNFLAGGED" : [(self.flagged, False)], "UNSEEN" : [(self.seen, False)], "OLD" : [(self.recent, False)], } complex_tests = {"BCC" : (self.string_contains, self.get("Bcc")), "SUBJECT" : (self.string_contains, self.get("Subject")), "CC" : (self.string_contains, self.get("Cc")), "BODY" : (self.string_contains, self.body()), "TO" : (self.string_contains, self.get("To")), "TEXT" : (self.string_contains, self.as_string()), "FROM" : (self.string_contains, self.get("From")), "SMALLER" : (operator.lt, len(self.as_string())), "LARGER" : (operator.gt, len(self.as_string())), "BEFORE" : (self.before, self.date), "ON" : (self.on, self.date), "SENTBEFORE" : (self.before, self.get("Date")), "SENTON" : (self.on, self.get("Date")), "SENTSINCE" : (self.since, self.get("Date")), "SINCE" : (self.since, self.date), } result = True test = None header = None header_field = None for c in criteria: if match_tests.has_key(c) and test is None and header is None: for (test, result) in match_tests[c]: result = result and (test == result) elif complex_tests.has_key(c) and test is None and header is None: test = complex_tests[c] elif test is not None and header is None: result = result and test[0](test[1], c) test = None elif c == "HEADER" and test is None: # the only criteria that uses the next _two_ elements header = c elif test is None and header is not None and header_field is None: header_field = c elif header is not None and header_field is not None and test is None: result = result and self.string_contains(self.get(header_field), c) header = None header_field = None return result """ Still to do: Messages with message sequence numbers corresponding to the specified message sequence number set UID Messages with unique identifiers corresponding to the specified unique identifier set. KEYWORD Messages with the specified keyword set. UNKEYWORD Messages that do not have the specified keyword set. NOT Messages that do not match the specified search key. OR Messages that match either search key. """ class IMAPFileMessageFactory(FileCorpus.FileMessageFactory): '''MessageFactory for IMAPFileMessage objects''' def create(self, key, directory): '''Create a message object from a filename in a directory''' return IMAPFileMessage(key, directory) class IMAPMailbox(cred.perspective.Perspective): __implements__ = (IMailbox,) def __init__(self, name, identity_name, id): cred.perspective.Perspective.__init__(self, name, identity_name) self.UID_validity = id self.listeners = [] def getUIDValidity(self): """Return the unique validity identifier for this mailbox.""" return self.UID_validity def addListener(self, listener): """Add a mailbox change listener.""" self.listeners.append(listener) def removeListener(self, listener): """Remove a mailbox change listener.""" self.listeners.remove(listener) class SpambayesMailbox(IMAPMailbox): def __init__(self, name, id, directory): IMAPMailbox.__init__(self, name, "spambayes", id) self.UID_validity = id ensureDir(directory) self.storage = FileCorpus.FileCorpus(IMAPFileMessageFactory(), directory, r"[0123456789]*") # UIDs are required to be strictly ascending. if len(self.storage.keys()) == 0: self.nextUID = 0 else: self.nextUID = long(self.storage.keys()[-1]) + 1 # Calculate initial recent and unseen counts # XXX Note that this will always end up with zero counts # XXX until the flags are persisted. self.unseen_count = 0 self.recent_count = 0 for msg in self.storage: if not msg.seen: self.unseen_count += 1 if msg.recent: self.recent_count += 1 def getUIDNext(self, increase=False): """Return the likely UID for the next message added to this mailbox.""" reply = str(self.nextUID) if increase: self.nextUID += 1 return reply def getUID(self, message): """Return the UID of a message in the mailbox.""" # Note that IMAP messages are 1-based, our messages are 0-based d = self.storage return long(d.keys()[message - 1]) def getFlags(self): """Return the flags defined in this mailbox.""" return ["\\Answered", "\\Flagged", "\\Deleted", "\\Seen", "\\Draft"] def getMessageCount(self): """Return the number of messages in this mailbox.""" return len(self.storage.keys()) def getRecentCount(self): """Return the number of messages with the 'Recent' flag.""" return self.recent_count def getUnseenCount(self): """Return the number of messages with the 'Unseen' flag.""" return self.unseen_count def isWriteable(self): """Get the read/write status of the mailbox.""" return True def destroy(self): """Called before this mailbox is deleted, permanently.""" # Our mailboxes cannot be deleted raise NotImplementedError def getHierarchicalDelimiter(self): """Get the character which delimits namespaces for in this mailbox.""" return '.' def requestStatus(self, names): """Return status information about this mailbox.""" answer = {} for request in names: request = request.upper() if request == "MESSAGES": answer[request] = self.getMessageCount() elif request == "RECENT": answer[request] = self.getRecentCount() elif request == "UIDNEXT": answer[request] = self.getUIDNext() elif request == "UIDVALIDITY": answer[request] = self.getUIDValidity() elif request == "UNSEEN": answer[request] = self.getUnseenCount() return answer def addMessage(self, message, flags=(), date=None): """Add the given message to this mailbox.""" msg = self.storage.makeMessage(self.getUIDNext(True)) msg.date = date msg.setPayload(message.read()) self.storage.addMessage(msg) self.store(MessageSet(long(msg.id), long(msg.id)), flags, 1, True) msg.recent = True msg.store() self.recent_count += 1 self.unseen_count += 1 for listener in self.listeners: listener.newMessages(self.getMessageCount(), self.getRecentCount()) d = defer.Deferred() reactor.callLater(0, d.callback, self.storage.keys().index(msg.id)) return d def expunge(self): """Remove all messages flagged \\Deleted.""" deleted_messages = [] for msg in self.storage: if msg.deleted: if not msg.seen: self.unseen_count -= 1 if msg.recent: self.recent_count -= 1 deleted_messages.append(long(msg.id)) self.storage.removeMessage(msg) if deleted_messages != []: for listener in self.listeners: listener.newMessages(self.getMessageCount(), self.getRecentCount()) return deleted_messages def search(self, query, uid): """Search for messages that meet the given query criteria. @type query: C{list} @param query: The search criteria @rtype: C{list} @return: A list of message sequence numbers or message UIDs which match the search criteria. """ if self.getMessageCount() == 0: return [] all_msgs = MessageSet(long(self.storage.keys()[0]), long(self.storage.keys()[-1])) matches = [] for id, msg in self._messagesIter(all_msgs, uid): for q in query: if msg.matches(q): matches.append(id) break return matches def _messagesIter(self, messages, uid): if uid: messages.last = long(self.storage.keys()[-1]) else: messages.last = self.getMessageCount() for id in messages: if uid: msg = self.storage.get(str(id)) else: msg = self.storage.get(str(self.getUID(id))) if msg is None: # Non-existant message. continue msg.load() yield (id, msg) def fetch(self, messages, uid): """Retrieve one or more messages.""" return self._messagesIter(messages, uid) def store(self, messages, flags, mode, uid): """Set the flags of one or more messages.""" stored_messages = {} for id, msg in self._messagesIter(messages, uid): if mode == 0: msg.clear_flags() value = True elif mode == -1: value = False elif mode == 1: value = True for flag in flags: if flag == '(' or flag == ')': continue if flag == "SEEN" and value == True and msg.seen == False: self.unseen_count -= 1 if flag == "SEEN" and value == False and msg.seen == True: self.unseen_count += 1 msg.set_flag(flag, value) stored_messages[id] = msg.flags() return stored_messages class Trainer(object): """Listens to a given mailbox and trains new messages as spam or ham.""" __implements__ = (IMailboxListener,) def __init__(self, mailbox, asSpam): self.mailbox = mailbox self.asSpam = asSpam def modeChanged(self, writeable): # We don't care pass def flagsChanged(self, newFlags): # We don't care pass def newMessages(self, exists, recent): # We don't get passed the actual message, or the id of # the message, of even the message number. We just get # the total number of new/recent messages. # However, this function should be called _every_ time # that a new message appears, so we should be able to # assume that the last message is the new one. # (We ignore the recent count) if exists is not None: id = self.mailbox.getUID(exists) msg = self.mailbox.storage[str(id)] msg.train(state.bayes, self.asSpam) class SpambayesAccount(MemoryAccount): """Account for Spambayes server.""" def __init__(self, id, ham, spam, unsure): MemoryAccount.__init__(self, id) self.mailboxes = {"SPAM" : spam, "UNSURE" : unsure, "TRAIN_AS_HAM" : ham} def select(self, name, readwrite=1): # 'INBOX' is a special case-insensitive name meaning the # primary mailbox for the user; we interpret this as an alias # for 'spam' if name.upper() == "INBOX": name = "SPAM" return MemoryAccount.select(self, name, readwrite) class SpambayesIMAPServer(IMAP4Server): IDENT = "Spambayes IMAP Server IMAP4rev1 Ready" def __init__(self, user_account): IMAP4Server.__init__(self) self.account = user_account def authenticateLogin(self, user, passwd): """Lookup the account associated with the given parameters.""" if user == options["imapserver", "username"] and \ passwd == options["imapserver", "password"]: return (IAccount, self.account, None) raise cred.error.UnauthorizedLogin() def connectionMade(self): state.activeIMAPSessions += 1 state.totalIMAPSessions += 1 IMAP4Server.connectionMade(self) def connectionLost(self, reason): state.activeIMAPSessions -= 1 IMAP4Server.connectionLost(self, reason) def do_CREATE(self, tag, args): """Creating new folders on the server is not permitted.""" self.sendNegativeResponse(tag, \ "Creation of new folders is not permitted") auth_CREATE = (do_CREATE, IMAP4Server.arg_astring) select_CREATE = auth_CREATE def do_DELETE(self, tag, args): """Deleting folders on the server is not permitted.""" self.sendNegativeResponse(tag, \ "Deletion of folders is not permitted") auth_DELETE = (do_DELETE, IMAP4Server.arg_astring) select_DELETE = auth_DELETE class OneParameterFactory(ServerFactory): """A factory that allows a single parameter to be passed to the created protocol.""" def buildProtocol(self, addr): """Create an instance of a subclass of Protocol, passing a single parameter.""" if self.parameter is not None: p = self.protocol(self.parameter) else: p = self.protocol() p.factory = self return p class MyBayesProxy(POP3ProxyBase): """Proxies between an email client and a POP3 server, redirecting mail to the imap server as necessary. It acts on the following POP3 commands: o RETR: o Adds the judgement header based on the raw headers and body of the message. """ intercept_message = 'From: "Spambayes" \n' \ 'Subject: Spambayes Intercept\n\nA message ' \ 'was intercepted by Spambayes (it scored %s).\n' \ '\nYou may find it in the Spam or Unsure ' \ 'folder.\n\n.\n' def __init__(self, clientSocket, serverName, serverPort, spam, unsure): POP3ProxyBase.__init__(self, clientSocket, serverName, serverPort) self.handlers = {'RETR': self.onRetr} state.totalSessions += 1 state.activeSessions += 1 self.isClosed = False self.spam_folder = spam self.unsure_folder = unsure def send(self, data): """Logs the data to the log file.""" if options["globals", "verbose"]: state.logFile.write(data) state.logFile.flush() try: return POP3ProxyBase.send(self, data) except socket.error: self.close() def recv(self, size): """Logs the data to the log file.""" data = POP3ProxyBase.recv(self, size) if options["globals", "verbose"]: state.logFile.write(data) state.logFile.flush() return data def close(self): # This can be called multiple times by async. if not self.isClosed: self.isClosed = True state.activeSessions -= 1 POP3ProxyBase.close(self) def onTransaction(self, command, args, response): """Takes the raw request and response, and returns the (possibly processed) response to pass back to the email client. """ handler = self.handlers.get(command, self.onUnknown) return handler(command, args, response) def onRetr(self, command, args, response): """Classifies the message. If the result is ham, then simply pass it through. If the result is an unsure or spam, move it to the appropriate IMAP folder.""" # Use '\n\r?\n' to detect the end of the headers in case of # broken emails that don't use the proper line separators. if re.search(r'\n\r?\n', response): # Break off the first line, which will be '+OK'. ok, messageText = response.split('\n', 1) prob = state.bayes.spamprob(tokenize(messageText)) if prob < options["Categorization", "ham_cutoff"]: # Return the +OK and the message with the header added. state.numHams += 1 return ok + "\n" + messageText elif prob > options["Categorization", "spam_cutoff"]: dest_folder = self.spam_folder state.numSpams += 1 else: dest_folder = self.unsure_folder state.numUnsure += 1 msg = StringIO.StringIO(messageText) date = imaplib.Time2Internaldate(time.time())[1:-1] dest_folder.addMessage(msg, (), date) # We have to return something, because the client is expecting # us to. We return a short message indicating that a message # was intercepted. return ok + "\n" + self.intercept_message % (prob,) else: # Must be an error response. return response def onUnknown(self, command, args, response): """Default handler; returns the server's response verbatim.""" return response class MyBayesProxyListener(Dibbler.Listener): """Listens for incoming email client connections and spins off MyBayesProxy objects to serve them. """ def __init__(self, serverName, serverPort, proxyPort, spam, unsure): proxyArgs = (serverName, serverPort, spam, unsure) Dibbler.Listener.__init__(self, proxyPort, MyBayesProxy, proxyArgs) print 'Listener on port %s is proxying %s:%d' % \ (_addressPortStr(proxyPort), serverName, serverPort) class IMAPState(State): def __init__(self): State.__init__(self) # Set up the extra statistics. self.totalIMAPSessions = 0 self.activeIMAPSessions = 0 def buildServerStrings(self): """After the server details have been set up, this creates string versions of the details, for display in the Status panel.""" self.serverPortString = str(self.imap_port) # Also build proxy strings State.buildServerStrings(self) state = IMAPState() # =================================================================== # __main__ driver. # =================================================================== def setup(): # Setup app, boxes, trainers and account proxyListeners = [] app = Application("SpambayesIMAPServer") spam_box = SpambayesMailbox("Spam", 0, options["imapserver", "spam_directory"]) unsure_box = SpambayesMailbox("Unsure", 1, options["imapserver", "unsure_directory"]) ham_train_box = SpambayesMailbox("TrainAsHam", 2, options["imapserver", "ham_directory"]) spam_trainer = Trainer(spam_box, True) ham_trainer = Trainer(ham_train_box, False) spam_box.addListener(spam_trainer) ham_train_box.addListener(ham_trainer) user_account = SpambayesAccount(options["imapserver", "username"], ham_train_box, spam_box, unsure_box) # add IMAP4 server f = OneParameterFactory() f.protocol = SpambayesIMAPServer f.parameter = user_account state.imap_port = options["imapserver", "port"] app.listenTCP(state.imap_port, f) # add POP3 proxy state.createWorkers() for (server, serverPort), proxyPort in zip(state.servers, state.proxyPorts): listener = MyBayesProxyListener(server, serverPort, proxyPort, spam_box, unsure_box) proxyListeners.append(listener) state.buildServerStrings() # add web interface httpServer = UserInterfaceServer(state.uiPort) serverUI = ServerUserInterface(state, _recreateState) httpServer.register(serverUI) return app def run(): # Read the arguments. try: opts, args = getopt.getopt(sys.argv[1:], 'hbd:D:u:') except getopt.error, msg: print >>sys.stderr, str(msg) + '\n\n' + __doc__ sys.exit() launchUI = False for opt, arg in opts: if opt == '-h': print >>sys.stderr, __doc__ sys.exit() elif opt == '-b': launchUI = True elif opt == '-d': # dbm file state.useDB = True options["Storage", "persistent_storage_file"] = arg elif opt == '-D': # pickle file state.useDB = False options["Storage", "persistent_storage_file"] = arg elif opt == '-u': state.uiPort = int(arg) # Let the user know what they are using... print get_version_string("IMAP Server") print "and engine %s.\n" % (get_version_string(),) # setup everything app = setup() # kick things off thread.start_new_thread(Dibbler.run, (launchUI,)) app.run(save=False) if __name__ == "__main__": run() --- NEW FILE: sb_server.py --- #!/usr/bin/env python """A POP3 proxy that works with classifier.py, and adds a simple X-Spambayes-Classification header (ham/spam/unsure) to each incoming email. You point pop3proxy at your POP3 server, and configure your email client to collect mail from the proxy then filter on the added header. Usage: pop3proxy.py [options] [ []] is the name of your real POP3 server is the port number of your real POP3 server, which defaults to 110. options: -h : Displays this help message. -d FILE : use the named DBM database file -D FILE : the the named Pickle database file -l port : proxy listens on this port number (default 110) -u port : User interface listens on this port number (default 8880; Browse http://localhost:8880/) -b : Launch a web browser showing the user interface. All command line arguments and switches take their default values from the [pop3proxy] and [html_ui] sections of bayescustomize.ini. For safety, and to help debugging, the whole POP3 conversation is written out to _pop3proxy.log for each run, if options["globals", "verbose"] is True. To make rebuilding the database easier, uploaded messages are appended to _pop3proxyham.mbox and _pop3proxyspam.mbox. """ # This module is part of the spambayes project, which is Copyright 2002 # The Python Software Foundation and is covered by the Python Software # Foundation license. __author__ = "Richie Hindle " __credits__ = "Tim Peters, Neale Pickett, Tim Stone, all the Spambayes folk." try: True, False except NameError: # Maintain compatibility with Python 2.2 True, False = 1, 0 todo = """ Web training interface: User interface improvements: o Once the pieces are on separate pages, make the paste box bigger. o Deployment: Windows executable? atlaxwin and ctypes? Or just webbrowser? o Save the stats (num classified, etc.) between sessions. o "Reload database" button. New features: o Online manual. o Links to project homepage, mailing list, etc. o List of words with stats (it would have to be paged!) a la SpamSieve. Code quality: o Cope with the email client timing out and closing the connection. Info: o Slightly-wordy index page; intro paragraph for each page. o In both stats and training results, report nham and nspam - warn if they're very different (for some value of 'very'). o "Links" section (on homepage?) to project homepage, mailing list, etc. Gimmicks: o Classify a web page given a URL. o Graphs. Of something. Who cares what? o NNTP proxy. o Zoe...! """ import os, sys, re, errno, getopt, time, traceback, socket, cStringIO from thread import start_new_thread from email.Header import Header import spambayes.message from spambayes import Dibbler from spambayes import storage from spambayes.FileCorpus import FileCorpus, ExpiryFileCorpus from spambayes.FileCorpus import FileMessageFactory, GzipFileMessageFactory from spambayes.Options import options from spambayes.UserInterface import UserInterfaceServer from spambayes.ProxyUI import ProxyUserInterface from spambayes.Version import get_version_string # Increase the stack size on MacOS X. Stolen from Lib/test/regrtest.py if sys.platform == 'darwin': try: import resource except ImportError: pass else: soft, hard = resource.getrlimit(resource.RLIMIT_STACK) newsoft = min(hard, max(soft, 1024*2048)) resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) # number to add to STAT length for each msg to fudge for spambayes headers HEADER_SIZE_FUDGE_FACTOR = 512 class ServerLineReader(Dibbler.BrighterAsyncChat): """An async socket that reads lines from a remote server and simply calls a callback with the data. The BayesProxy object can't connect to the real POP3 server and talk to it synchronously, because that would block the process.""" lineCallback = None def __init__(self, serverName, serverPort, lineCallback): Dibbler.BrighterAsyncChat.__init__(self) self.lineCallback = lineCallback self.request = '' self.set_terminator('\r\n') self.create_socket(socket.AF_INET, socket.SOCK_STREAM) try: self.connect((serverName, serverPort)) except socket.error, e: error = "Can't connect to %s:%d: %s" % (serverName, serverPort, e) print >>sys.stderr, error self.lineCallback('-ERR %s\r\n' % error) self.lineCallback('') # "The socket's been closed." self.close() def collect_incoming_data(self, data): self.request = self.request + data def found_terminator(self): self.lineCallback(self.request + '\r\n') self.request = '' def handle_close(self): self.lineCallback('') self.close() class POP3ProxyBase(Dibbler.BrighterAsyncChat): """An async dispatcher that understands POP3 and proxies to a POP3 server, calling `self.onTransaction(request, response)` for each transaction. Responses are not un-byte-stuffed before reaching self.onTransaction() (they probably should be for a totally generic POP3ProxyBase class, but BayesProxy doesn't need it and it would mean re-stuffing them afterwards). self.onTransaction() should return the response to pass back to the email client - the response can be the verbatim response or a processed version of it. The special command 'KILL' kills it (passing a 'QUIT' command to the server). """ def __init__(self, clientSocket, serverName, serverPort): Dibbler.BrighterAsyncChat.__init__(self, clientSocket) self.request = '' self.response = '' self.set_terminator('\r\n') self.command = '' # The POP3 command being processed... self.args = [] # ...and its arguments self.isClosing = False # Has the server closed the socket? self.seenAllHeaders = False # For the current RETR or TOP self.startTime = 0 # (ditto) self.serverSocket = ServerLineReader(serverName, serverPort, self.onServerLine) def onTransaction(self, command, args, response): """Overide this. Takes the raw request and the response, and returns the (possibly processed) response to pass back to the email client. """ raise NotImplementedError def onServerLine(self, line): """A line of response has been received from the POP3 server.""" isFirstLine = not self.response self.response = self.response + line # Is this the line that terminates a set of headers? self.seenAllHeaders = self.seenAllHeaders or line in ['\r\n', '\n'] # Has the server closed its end of the socket? if not line: self.isClosing = True # If we're not processing a command, just echo the response. if not self.command: self.push(self.response) self.response = '' # Time out after 30 seconds for message-retrieval commands if # all the headers are down. The rest of the message will proxy # straight through. if self.command in ['TOP', 'RETR'] and \ self.seenAllHeaders and time.time() > self.startTime + 30: self.onResponse() self.response = '' # If that's a complete response, handle it. elif not self.isMultiline() or line == '.\r\n' or \ (isFirstLine and line.startswith('-ERR')): self.onResponse() self.response = '' def isMultiline(self): """Returns True if the request should get a multiline response (assuming the response is positive). """ if self.command in ['USER', 'PASS', 'APOP', 'QUIT', 'STAT', 'DELE', 'NOOP', 'RSET', 'KILL']: return False elif self.command in ['RETR', 'TOP', 'CAPA']: return True elif self.command in ['LIST', 'UIDL']: return len(self.args) == 0 else: # Assume that an unknown command will get a single-line # response. This should work for errors and for POP-AUTH, # and is harmless even for multiline responses - the first # line will be passed to onTransaction and ignored, then the # rest will be proxied straight through. return False def collect_incoming_data(self, data): """Asynchat override.""" self.request = self.request + data def found_terminator(self): """Asynchat override.""" verb = self.request.strip().upper() if verb == 'KILL': self.socket.shutdown(2) self.close() raise SystemExit elif verb == 'CRASH': # For testing x = 0 y = 1/x self.serverSocket.push(self.request + '\r\n') if self.request.strip() == '': # Someone just hit the Enter key. self.command = '' self.args = [] else: # A proper command. splitCommand = self.request.strip().split() self.command = splitCommand[0].upper() self.args = splitCommand[1:] self.startTime = time.time() self.request = '' def onResponse(self): # We don't support pipelining, so if the command is CAPA and the # response includes PIPELINING, hack out that line of the response. if self.command == 'CAPA': pipelineRE = r'(?im)^PIPELINING[^\n]*\n' self.response = re.sub(pipelineRE, '', self.response) # Pass the request and the raw response to the subclass and # send back the cooked response. if self.response: cooked = self.onTransaction(self.command, self.args, self.response) self.push(cooked) # If onServerLine() decided that the server has closed its # socket, close this one when the response has been sent. if self.isClosing: self.close_when_done() # Reset. self.command = '' self.args = [] self.isClosing = False self.seenAllHeaders = False class BayesProxyListener(Dibbler.Listener): """Listens for incoming email client connections and spins off BayesProxy objects to serve them. """ def __init__(self, serverName, serverPort, proxyPort): proxyArgs = (serverName, serverPort) Dibbler.Listener.__init__(self, proxyPort, BayesProxy, proxyArgs) print 'Listener on port %s is proxying %s:%d' % \ (_addressPortStr(proxyPort), serverName, serverPort) class BayesProxy(POP3ProxyBase): """Proxies between an email client and a POP3 server, inserting judgement headers. It acts on the following POP3 commands: o STAT: o Adds the size of all the judgement headers to the maildrop size. o LIST: o With no message number: adds the size of an judgement header to the message size for each message in the scan listing. o With a message number: adds the size of an judgement header to the message size. o RETR: o Adds the judgement header based on the raw headers and body of the message. o TOP: o Adds the judgement header based on the raw headers and as much of the body as the TOP command retrieves. This can mean that the header might have a different value for different calls to TOP, or for calls to TOP vs. calls to RETR. I'm assuming that the email client will either not make multiple calls, or will cope with the headers being different. o USER: o Does no processing based on the USER command itself, but expires any old messages in the three caches. """ def __init__(self, clientSocket, serverName, serverPort): POP3ProxyBase.__init__(self, clientSocket, serverName, serverPort) self.handlers = {'STAT': self.onStat, 'LIST': self.onList, 'RETR': self.onRetr, 'TOP': self.onTop, 'USER': self.onUser} state.totalSessions += 1 state.activeSessions += 1 self.isClosed = False def send(self, data): """Logs the data to the log file.""" if options["globals", "verbose"]: state.logFile.write(data) state.logFile.flush() try: return POP3ProxyBase.send(self, data) except socket.error: # The email client has closed the connection - 40tude Dialog # does this immediately after issuing a QUIT command, # without waiting for the response. self.close() def recv(self, size): """Logs the data to the log file.""" data = POP3ProxyBase.recv(self, size) if options["globals", "verbose"]: state.logFile.write(data) state.logFile.flush() return data def close(self): # This can be called multiple times by async. if not self.isClosed: self.isClosed = True state.activeSessions -= 1 POP3ProxyBase.close(self) def onTransaction(self, command, args, response): """Takes the raw request and response, and returns the (possibly processed) response to pass back to the email client. """ handler = self.handlers.get(command, self.onUnknown) return handler(command, args, response) def onStat(self, command, args, response): """Adds the size of all the judgement headers to the maildrop size.""" match = re.search(r'^\+OK\s+(\d+)\s+(\d+)(.*)\r\n', response) if match: count = int(match.group(1)) size = int(match.group(2)) + HEADER_SIZE_FUDGE_FACTOR * count return '+OK %d %d%s\r\n' % (count, size, match.group(3)) else: return response def onList(self, command, args, response): """Adds the size of an judgement header to the message size(s).""" if response.count('\r\n') > 1: # Multiline: all lines but the first contain a message size. lines = response.split('\r\n') outputLines = [lines[0]] for line in lines[1:]: match = re.search(r'^(\d+)\s+(\d+)', line) if match: number = int(match.group(1)) size = int(match.group(2)) + HEADER_SIZE_FUDGE_FACTOR line = "%d %d" % (number, size) outputLines.append(line) return '\r\n'.join(outputLines) else: # Single line. match = re.search(r'^\+OK\s+(\d+)\s+(\d+)(.*)\r\n', response) if match: messageNumber = match.group(1) size = int(match.group(2)) + HEADER_SIZE_FUDGE_FACTOR trailer = match.group(3) return "+OK %s %s%s\r\n" % (messageNumber, size, trailer) else: return response def onRetr(self, command, args, response): """Adds the judgement header based on the raw headers and body of the message.""" # Use '\n\r?\n' to detect the end of the headers in case of # broken emails that don't use the proper line separators. if re.search(r'\n\r?\n', response): # Remove the trailing .\r\n before passing to the email parser. # Thanks to Scott Schlesier for this fix. terminatingDotPresent = (response[-4:] == '\n.\r\n') if terminatingDotPresent: response = response[:-3] # Break off the first line, which will be '+OK'. ok, messageText = response.split('\n', 1) try: msg = spambayes.message.SBHeaderMessage() msg.setPayload(messageText) msg.setId(state.getNewMessageName()) # Now find the spam disposition and add the header. (prob, clues) = state.bayes.spamprob(msg.asTokens(),\ evidence=True) msg.addSBHeaders(prob, clues) # Check for "RETR" or "TOP N 99999999" - fetchmail without # the 'fetchall' option uses the latter to retrieve messages. if (command == 'RETR' or (command == 'TOP' and len(args) == 2 and args[1] == '99999999')): cls = msg.GetClassification() if cls == options["Headers", "header_ham_string"]: state.numHams += 1 elif cls == options["Headers", "header_spam_string"]: state.numSpams += 1 else: state.numUnsure += 1 # Suppress caching of "Precedence: bulk" or # "Precedence: list" ham if the options say so. isSuppressedBulkHam = \ (cls == options["Headers", "header_ham_string"] and options["pop3proxy", "no_cache_bulk_ham"] and msg.get('precedence') in ['bulk', 'list']) # Suppress large messages if the options say so. size_limit = options["pop3proxy", "no_cache_large_messages"] isTooBig = size_limit > 0 and \ len(messageText) > size_limit # Cache the message. Don't pollute the cache with test # messages or suppressed bulk ham. if (not state.isTest and options["pop3proxy", "cache_messages"] and not isSuppressedBulkHam and not isTooBig): # Write the message into the Unknown cache. message = state.unknownCorpus.makeMessage(msg.getId()) message.setSubstance(msg.as_string()) state.unknownCorpus.addMessage(message) # We'll return the message with the headers added. We take # all the headers from the SBHeaderMessage, but take the body # directly from the POP3 conversation, because the # SBHeaderMessage might have "fixed" a partial message by # appending a closing boundary separator. Remember we can # be dealing with partial message here because of the timeout # code in onServerLine. headers = [] for name, value in msg.items(): header = "%s: %s" % (name, value) headers.append(re.sub(r'\r?\n', '\r\n', header)) body = re.split(r'\n\r?\n', messageText, 1)[1] messageText = "\r\n".join(headers) + "\r\n\r\n" + body except: # Something nasty happened while parsing or classifying - # report the exception in a hand-appended header and recover. # This is one case where an unqualified 'except' is OK, 'cos # anything's better than destroying people's email... stream = cStringIO.StringIO() traceback.print_exc(None, stream) details = stream.getvalue() # Build the header. This will strip leading whitespace from # the lines, so we add a leading dot to maintain indentation. detailLines = details.strip().split('\n') dottedDetails = '\n.'.join(detailLines) headerName = 'X-Spambayes-Exception' header = Header(dottedDetails, header_name=headerName) # Insert the header, converting email.Header's '\n' line # breaks to POP3's '\r\n'. headers, body = re.split(r'\n\r?\n', messageText, 1) header = re.sub(r'\r?\n', '\r\n', str(header)) headers += "\n%s: %s\r\n\r\n" % (headerName, header) messageText = headers + body # Print the exception and a traceback. print >>sys.stderr, details # Restore the +OK and the POP3 .\r\n terminator if there was one. retval = ok + "\n" + messageText if terminatingDotPresent: retval += '.\r\n' return retval else: # Must be an error response. return response def onTop(self, command, args, response): """Adds the judgement header based on the raw headers and as much of the body as the TOP command retrieves.""" # Easy (but see the caveat in BayesProxy.__doc__). return self.onRetr(command, args, response) def onUser(self, command, args, response): """Spins off three separate threads that expires any old messages in the three caches, but does not do any processing of the USER command itself.""" start_new_thread(state.spamCorpus.removeExpiredMessages, ()) start_new_thread(state.hamCorpus.removeExpiredMessages, ()) start_new_thread(state.unknownCorpus.removeExpiredMessages, ()) return response def onUnknown(self, command, args, response): """Default handler; returns the server's response verbatim.""" return response # This keeps the global state of the module - the command-line options, # statistics like how many mails have been classified, the handle of the # log file, the Classifier and FileCorpus objects, and so on. class State: def __init__(self): """Initialises the State object that holds the state of the app. The default settings are read from Options.py and bayescustomize.ini and are then overridden by the command-line processing code in the __main__ code below.""" # Open the log file. if options["globals", "verbose"]: self.logFile = open('_pop3proxy.log', 'wb', 0) self.servers = [] self.proxyPorts = [] if options["pop3proxy", "remote_servers"]: for server in options["pop3proxy", "remote_servers"]: server = server.strip() if server.find(':') > -1: server, port = server.split(':', 1) else: port = '110' self.servers.append((server, int(port))) if options["pop3proxy", "listen_ports"]: splitPorts = options["pop3proxy", "listen_ports"] self.proxyPorts = map(_addressAndPort, splitPorts) if len(self.servers) != len(self.proxyPorts): print "pop3proxy_servers & pop3proxy_ports are different lengths!" sys.exit() # Load up the other settings from Option.py / bayescustomize.ini self.useDB = options["Storage", "persistent_use_database"] self.uiPort = options["html_ui", "port"] self.launchUI = options["html_ui", "launch_browser"] self.gzipCache = options["pop3proxy", "cache_use_gzip"] self.cacheExpiryDays = options["pop3proxy", "cache_expiry_days"] self.runTestServer = False self.isTest = False # Set up the statistics. self.totalSessions = 0 self.activeSessions = 0 self.numSpams = 0 self.numHams = 0 self.numUnsure = 0 # Unique names for cached messages - see `getNewMessageName()` below. self.lastBaseMessageName = '' self.uniquifier = 2 def buildServerStrings(self): """After the server details have been set up, this creates string versions of the details, for display in the Status panel.""" serverStrings = ["%s:%s" % (s, p) for s, p in self.servers] self.serversString = ', '.join(serverStrings) self.proxyPortsString = ', '.join(map(_addressPortStr, self.proxyPorts)) def createWorkers(self): """Using the options that were initialised in __init__ and then possibly overridden by the driver code, create the Bayes object, the Corpuses, the Trainers and so on.""" print "Loading database...", if self.isTest: self.useDB = True options["Storage", "persistent_storage_file"] = \ '_pop3proxy_test.pickle' # This is never saved. filename = options["Storage", "persistent_storage_file"] filename = os.path.expanduser(filename) self.bayes = storage.open_storage(filename, self.useDB) # Don't set up the caches and training objects when running the self-test, # so as not to clutter the filesystem. if not self.isTest: def ensureDir(dirname): try: os.mkdir(dirname) except OSError, e: if e.errno != errno.EEXIST: raise # Create/open the Corpuses. Use small cache sizes to avoid hogging # lots of memory. map(ensureDir, [options["pop3proxy", "spam_cache"], options["pop3proxy", "ham_cache"], options["pop3proxy", "unknown_cache"]]) if self.gzipCache: factory = GzipFileMessageFactory() else: factory = FileMessageFactory() age = options["pop3proxy", "cache_expiry_days"]*24*60*60 self.spamCorpus = ExpiryFileCorpus(age, factory, options["pop3proxy", "spam_cache"], '[0123456789\-]*', cacheSize=20) self.hamCorpus = ExpiryFileCorpus(age, factory, options["pop3proxy", "ham_cache"], '[0123456789\-]*', cacheSize=20) self.unknownCorpus = ExpiryFileCorpus(age, factory, options["pop3proxy", "unknown_cache"], '[0123456789\-]*', cacheSize=20) # Given that (hopefully) users will get to the stage # where they do not need to do any more regular training to # be satisfied with spambayes' performance, we expire old # messages from not only the trained corpora, but the unknown # as well. self.spamCorpus.removeExpiredMessages() self.hamCorpus.removeExpiredMessages() self.unknownCorpus.removeExpiredMessages() # Create the Trainers. self.spamTrainer = storage.SpamTrainer(self.bayes) self.hamTrainer = storage.HamTrainer(self.bayes) self.spamCorpus.addObserver(self.spamTrainer) self.hamCorpus.addObserver(self.hamTrainer) def getNewMessageName(self): # The message name is the time it arrived, with a uniquifier # appended if two arrive within one clock tick of each other. messageName = "%10.10d" % long(time.time()) if messageName == self.lastBaseMessageName: messageName = "%s-%d" % (messageName, self.uniquifier) self.uniquifier += 1 else: self.lastBaseMessageName = messageName self.uniquifier = 2 return messageName # Option-parsing helper functions def _addressAndPort(s): """Decode a string representing a port to bind to, with optional address.""" s = s.strip() if ':' in s: addr, port = s.split(':') return addr, int(port) else: return '', int(s) def _addressPortStr((addr, port)): """Encode a string representing a port to bind to, with optional address.""" if not addr: return str(port) else: return '%s:%d' % (addr, port) state = State() proxyListeners = [] def _createProxies(servers, proxyPorts): """Create BayesProxyListeners for all the given servers.""" for (server, serverPort), proxyPort in zip(servers, proxyPorts): listener = BayesProxyListener(server, serverPort, proxyPort) proxyListeners.append(listener) def _recreateState(): global state state = State() # Close the existing listeners and create new ones. This won't # affect any running proxies - once a listener has created a proxy, # that proxy is then independent of it. for proxy in proxyListeners: proxy.close() del proxyListeners[:] prepare(state) _createProxies(state.servers, state.proxyPorts) return state def main(servers, proxyPorts, uiPort, launchUI): """Runs the proxy forever or until a 'KILL' command is received or someone hits Ctrl+Break.""" _createProxies(servers, proxyPorts) httpServer = UserInterfaceServer(uiPort) proxyUI = ProxyUserInterface(state, _recreateState) httpServer.register(proxyUI) Dibbler.run(launchBrowser=launchUI) def prepare(state): # Do whatever we've been asked to do... state.createWorkers() # Launch any SMTP proxies. Note that if the user hasn't specified any # SMTP proxy information in their configuration, then nothing will # happen. import sb_smtpproxy servers, proxyPorts = sb_smtpproxy.LoadServerInfo() proxyListeners.extend(sb_smtpproxy.CreateProxies(servers, proxyPorts, state)) # setup info for the web interface state.buildServerStrings() def start(state): # kick everything off main(state.servers, state.proxyPorts, state.uiPort, state.launchUI) def stop(state): # Shutdown as though through the web UI. This will save the DB, allow # any open proxy connections to complete, etc. from urllib2 import urlopen from urllib import urlencode urlopen('http://localhost:%d/save' % state.uiPort, urlencode({'how': 'Save & shutdown'})).read() # =================================================================== # __main__ driver. # =================================================================== def run(): # Read the arguments. try: opts, args = getopt.getopt(sys.argv[1:], 'hbpsd:D:l:u:') except getopt.error, msg: print >>sys.stderr, str(msg) + '\n\n' + __doc__ sys.exit() runSelfTest = False for opt, arg in opts: if opt == '-h': print >>sys.stderr, __doc__ sys.exit() elif opt == '-b': state.launchUI = True elif opt == '-d': # dbm file state.useDB = True options["Storage", "persistent_storage_file"] = arg elif opt == '-D': # pickle file state.useDB = False options["Storage", "persistent_storage_file"] = arg elif opt == '-p': # dead option print >>sys.stderr, "-p option is no longer supported, use -D\n" print >>sys.stderr, __doc__ sys.exit() elif opt == '-l': state.proxyPorts = [_addressAndPort(arg)] elif opt == '-u': state.uiPort = int(arg) # Let the user know what they are using... print get_version_string("POP3 Proxy") print "and engine %s.\n" % (get_version_string(),) prepare(state=state) if 0 <= len(args) <= 2: # Normal usage, with optional server name and port number. if len(args) == 1: state.servers = [(args[0], 110)] elif len(args) == 2: state.servers = [(args[0], int(args[1]))] # Default to listening on port 110 for command-line-specified servers. if len(args) > 0 and state.proxyPorts == []: state.proxyPorts = [('', 110)] start(state=state) else: print >>sys.stderr, __doc__ if __name__ == '__main__': run() --- NEW FILE: sb_smtpproxy.py --- #!/usr/bin/env python """A SMTP proxy to train a Spambayes database. You point SMTP Proxy at your SMTP server(s) and configure your email client(s) to send mail through the proxy (i.e. usually this means you use localhost as the outgoing server). To setup, enter appropriate values in your Spambayes configuration file in the "SMTP Proxy" section (in particular: "remote_servers", "listen_ports", and "use_cached_message"). This configuration can also be carried out via the web user interface offered by POP3 Proxy and IMAP Filter. To use, simply forward/bounce mail that you wish to train to the appropriate address (defaults to spambayes_spam@localhost and spambayes_ham@localhost). All other mail is sent normally. (Note that IMAP Filter and POP3 Proxy users should not execute this script; launching of SMTP Proxy will be taken care of by those applicatons). There are two main forms of operation. With both, mail to two (user-configurable) email addresses is intercepted by the proxy (and is *not* sent to the SMTP server) and used as training data for a Spambayes database. All other mail is simply relayed to the SMTP server. If the "use_cached_message" option is False, the proxy uses the message sent as training data. This option is suitable for those not using POP3 Proxy or IMAP Filter, or for those that are confident that their mailer will forward/bounce messages in an unaltered form. If the "use_cached_message" option is True, the proxy examines the message for a unique spambayes identification number. It then tries to find this message in the pop3proxy caches and on the imap servers. It then retrieves the message from the cache/server and uses *this* as the training data. This method is suitable for those using POP3 Proxy and/or IMAP Filter, and avoids any potential problems with the mailer altering messages before forwarding/bouncing them. Usage: smtpproxy [options] note: option values with spaces must be enclosed in double quotes options: -d dbname : pickled training database filename -D dbname : dbm training database filename -h : help -v : verbose mode """ # This module is part of the spambayes project, which is Copyright 2002-3 # The Python Software Foundation and is covered by the Python Software # Foundation license. __author__ = "Tony Meyer " __credits__ = "Tim Stone, all the Spambayes folk." try: True, False except NameError: # Maintain compatibility with Python 2.2 True, False = 1, 0 todo = """ o It would be nice if spam/ham could be bulk forwarded to the proxy, rather than one by one. This would require separating the different messages and extracting the correct ids. Simply changing to find *all* the ids in a message, rather than stopping after one *might* work, but I don't really know. Richie Hindle suggested something along these lines back in September '02. o Suggestions? Testing: o Test with as many clients as possible to check that the id is correctly extracted from the forwarded/bounced message. MUA information: A '*' in the Header column signifies that the smtpproxy can extract the id from the headers only. A '*' in the Body column signifies that the smtpproxy can extract the id from the body of the message, if it is there. Header Body *** Windows 2000 MUAs *** Eudora 5.2 Forward * * Eudora 5.2 Redirect * Netscape Messenger (4.7) Forward (inline) * * Netscape Messenger (4.7) Forward (quoted) Plain * Netscape Messenger (4.7) Forward (quoted) HTML * Netscape Messenger (4.7) Forward (quoted) Plain & HTML * Netscape Messenger (4.7) Forward (attachment) Plain * * Netscape Messenger (4.7) Forward (attachment) HTML * * Netscape Messenger (4.7) Forward (attachment) Plain & HTML * * Outlook Express 6 Forward HTML (Base64) * Outlook Express 6 Forward HTML (None) * Outlook Express 6 Forward HTML (QP) * Outlook Express 6 Forward Plain (Base64) * Outlook Express 6 Forward Plain (None) * Outlook Express 6 Forward Plain (QP) * Outlook Express 6 Forward Plain (uuencoded) * http://www.endymion.com/products/mailman Forward * M2 (Opera Mailer 7.01) Forward * M2 (Opera Mailer 7.01) Redirect * * The Bat! 1.62i Forward (RFC Headers not visible) * The Bat! 1.62i Forward (RFC Headers visible) * * The Bat! 1.62i Redirect * The Bat! 1.62i Alternative Forward * * The Bat! 1.62i Custom Template * * AllegroMail 2.5.0.2 Forward * AllegroMail 2.5.0.2 Redirect * PocoMail 2.6.3 Bounce * PocoMail 2.6.3 Bounce * Pegasus Mail 4.02 Forward (all headers option set) * * Pegasus Mail 4.02 Forward (all headers option not set) * Calypso 3 Forward * Calypso 3 Redirect * * Becky! 2.05.10 Forward * Becky! 2.05.10 Redirect * Becky! 2.05.10 Redirect as attachment * * Mozilla Mail 1.2.1 Forward (attachment) * * Mozilla Mail 1.2.1 Forward (inline, plain) *1 * Mozilla Mail 1.2.1 Forward (inline, plain & html) *1 * Mozilla Mail 1.2.1 Forward (inline, html) *1 * *1 The header method will only work if auto-include original message is set, and if view all headers is true. """ import string import re import socket import asyncore import asynchat import getopt import sys import os from spambayes import Dibbler from spambayes import storage from spambayes.message import sbheadermessage_from_string from spambayes.tokenizer import textparts from spambayes.tokenizer import try_to_repair_damaged_base64 from spambayes.Options import options from sb_server import _addressPortStr, ServerLineReader from sb_server import _addressAndPort class SMTPProxyBase(Dibbler.BrighterAsyncChat): """An async dispatcher that understands SMTP and proxies to a SMTP server, calling `self.onTransaction(command, args)` for each transaction. self.onTransaction() should return the command to pass to the proxied server - the command can be the verbatim command or a processed version of it. The special command 'KILL' kills it (passing a 'QUIT' command to the server). """ def __init__(self, clientSocket, serverName, serverPort): Dibbler.BrighterAsyncChat.__init__(self, clientSocket) self.request = '' self.set_terminator('\r\n') self.command = '' # The SMTP command being processed... self.args = '' # ...and its arguments self.isClosing = False # Has the server closed the socket? self.inData = False self.data = "" self.blockData = False self.serverSocket = ServerLineReader(serverName, serverPort, self.onServerLine) def onTransaction(self, command, args): """Overide this. Takes the raw command and returns the (possibly processed) command to pass to the email client.""" raise NotImplementedError def onProcessData(self, data): """Overide this. Takes the raw data and returns the (possibly processed) data to pass back to the email client.""" raise NotImplementedError def onServerLine(self, line): """A line of response has been received from the SMTP server.""" # Has the server closed its end of the socket? if not line: self.isClosing = True # We don't process the return, just echo the response. self.push(line) self.onResponse() def collect_incoming_data(self, data): """Asynchat override.""" self.request = self.request + data def found_terminator(self): """Asynchat override.""" verb = self.request.strip().upper() if verb == 'KILL': self.socket.shutdown(2) self.close() raise SystemExit if self.request.strip() == '': # Someone just hit the Enter key. self.command = self.args = '' else: # A proper command. if self.request[:10].upper() == "MAIL FROM:": splitCommand = self.request.split(":", 1) elif self.request[:8].upper() == "RCPT TO:": splitCommand = self.request.split(":", 1) else: splitCommand = self.request.strip().split(None, 1) self.command = splitCommand[0] self.args = splitCommand[1:] if self.inData == True: self.data += self.request + '\r\n' if self.request == ".": self.inData = False cooked = self.onProcessData(self.data) self.data = "" if self.blockData == False: self.serverSocket.push(cooked) else: self.push("250 OK\r\n") else: cooked = self.onTransaction(self.command, self.args) if cooked is not None: self.serverSocket.push(cooked + '\r\n') self.command = self.args = self.request = '' def onResponse(self): # If onServerLine() decided that the server has closed its # socket, close this one when the response has been sent. if self.isClosing: self.close_when_done() # Reset. self.command = '' self.args = '' self.isClosing = False class BayesSMTPProxyListener(Dibbler.Listener): """Listens for incoming email client connections and spins off BayesSMTPProxy objects to serve them.""" def __init__(self, serverName, serverPort, proxyPort, trainer): proxyArgs = (serverName, serverPort, trainer) Dibbler.Listener.__init__(self, proxyPort, BayesSMTPProxy, proxyArgs) print 'SMTP Listener on port %s is proxying %s:%d' % \ (_addressPortStr(proxyPort), serverName, serverPort) class BayesSMTPProxy(SMTPProxyBase): """Proxies between an email client and a SMTP server, inserting judgement headers. It acts on the following SMTP commands: o RCPT TO: o Checks if the recipient address matches the key ham or spam addresses, and if so notes this and does not forward a command to the proxied server. In all other cases simply passes on the verbatim command. o DATA: o Notes that we are in the data section. If (from the RCPT TO information) we are receiving a ham/spam message to train on, then do not forward the command on. Otherwise forward verbatim. Any other commands are merely passed on verbatim to the server. """ def __init__(self, clientSocket, serverName, serverPort, trainer): SMTPProxyBase.__init__(self, clientSocket, serverName, serverPort) self.handlers = {'RCPT TO': self.onRcptTo, 'DATA': self.onData, 'MAIL FROM': self.onMailFrom} self.trainer = trainer self.isClosed = False self.train_as_ham = False self.train_as_spam = False def send(self, data): try: return SMTPProxyBase.send(self, data) except socket.error: # The email client has closed the connection - 40tude Dialog # does this immediately after issuing a QUIT command, # without waiting for the response. self.close() def close(self): # This can be called multiple times by async. if not self.isClosed: self.isClosed = True SMTPProxyBase.close(self) def stripAddress(self, address): """ Strip the leading & trailing <> from an address. Handy for getting FROM: addresses. """ if '<' in address: start = string.index(address, '<') + 1 end = string.index(address, '>') return address[start:end] else: return address def splitTo(self, address): """Return 'address' as undressed (host, fulladdress) tuple. Handy for use with TO: addresses.""" start = string.index(address, '<') + 1 sep = string.index(address, '@') + 1 end = string.index(address, '>') return (address[sep:end], address[start:end],) def onTransaction(self, command, args): handler = self.handlers.get(command.upper(), self.onUnknown) return handler(command, args) def onProcessData(self, data): if self.train_as_spam: self.trainer.train(data, True) self.train_as_spam = False return "" elif self.train_as_ham: self.trainer.train(data, False) self.train_as_ham = False return "" return data def onRcptTo(self, command, args): toHost, toFull = self.splitTo(args[0]) if toFull == options["smtpproxy", "spam_address"]: self.train_as_spam = True self.train_as_ham = False self.blockData = True self.push("250 OK\r\n") return None elif toFull == options["smtpproxy", "ham_address"]: self.train_as_ham = True self.train_as_spam = False self.blockData = True self.push("250 OK\r\n") return None else: self.blockData = False return "%s:%s" % (command, ' '.join(args)) def onData(self, command, args): self.inData = True if self.train_as_ham == True or self.train_as_spam == True: self.push("250 OK\r\n") return None rv = command for arg in args: rv += ' ' + arg return rv def onMailFrom(self, command, args): """Just like the default handler, but has the necessary colon.""" rv = "%s:%s" % (command, ' '.join(args)) return rv def onUnknown(self, command, args): """Default handler.""" return self.request class SMTPTrainer(object): def __init__(self, classifier, state=None, imap=None): self.classifier = classifier self.state = state self.imap = imap def extractSpambayesID(self, data): msg = message_from_string(data) # The nicest MUA is one that forwards the header intact. id = msg.get(options["Headers", "mailid_header_name"]) if id is not None: return id # Some MUAs will put it in the body somewhere, while others will # put it in an attached MIME message. id = self._find_id_in_text(msg.as_string()) if id is not None: return id # the message might be encoded for part in textparts(msg): # Decode, or take it as-is if decoding fails. try: text = part.get_payload(decode=True) except: text = part.get_payload(decode=False) if text is not None: text = try_to_repair_damaged_base64(text) if text is not None: id = self._find_id_in_text(text) return id return None header_pattern = re.escape(options["Headers", "mailid_header_name"]) # A MUA might enclose the id in a table, thus the convoluted re pattern # (Mozilla Mail does this with inline html) header_pattern += r":\s*(\\s*\\s*)?([\d\-]+)" header_re = re.compile(header_pattern) def _find_id_in_text(self, text): mo = self.header_re.search(text) if mo is None: return None return mo.group(2) def train(self, msg, isSpam): try: use_cached = options["smtpproxy", "use_cached_message"] except KeyError: use_cached = True if use_cached: id = self.extractSpambayesID(msg) if id is None: print "Could not extract id" return self.train_cached_message(id, isSpam) # Otherwise, train on the forwarded/bounced message. msg = sbheadermessage_from_string(msg) id = msg.setIdFromPayload() msg.delSBHeaders() if id is None: # No id, so we don't have any reliable method of remembering # information about this message, so we just assume that it # hasn't been trained before. We could generate some sort of # checksum for the message and use that as an id (this would # mean that we didn't need to store the id with the message) # but that might be a little unreliable. self.classifier.learn(msg.asTokens(), isSpam) else: if msg.GetTrained() == (not isSpam): self.classifier.unlearn(msg.asTokens(), not isSpam) msg.RememberTrained(None) if msg.GetTrained() is None: self.classifier.learn(msg.asTokens(), isSpam) msg.RememberTrained(isSpam) def train_cached_message(self, id, isSpam): if not self.train_message_in_pop3proxy_cache(id, isSpam) and \ not self.train_message_on_imap_server(id, isSpam): print "Could not find message (%s); perhaps it was " + \ "deleted from the POP3Proxy cache or the IMAP " + \ "server. This means that no training was done." % (id, ) def train_message_in_pop3proxy_cache(self, id, isSpam): if self.state is None: return False sourceCorpus = None for corpus in [self.state.unknownCorpus, self.state.hamCorpus, self.state.spamCorpus]: if corpus.get(id) is not None: sourceCorpus = corpus break if corpus is None: return False if isSpam == True: targetCorpus = self.state.spamCorpus else: targetCorpus = self.state.hamCorpus targetCorpus.takeMessage(id, sourceCorpus) self.classifier.store() def train_message_on_imap_server(self, id, isSpam): if self.imap is None: return False msg = self.imap.FindMessage(id) if msg is None: return False if msg.GetTrained() == (not isSpam): msg.get_substance() msg.delSBHeaders() self.classifier.unlearn(msg.asTokens(), not isSpam) msg.RememberTrained(None) if msg.GetTrained() is None: msg.get_substance() msg.delSBHeaders() self.classifier.learn(msg.asTokens(), isSpam) msg.RememberTrained(isSpam) def LoadServerInfo(): # Load the proxy settings servers = [] proxyPorts = [] if options["smtpproxy", "remote_servers"]: for server in options["smtpproxy", "remote_servers"]: server = server.strip() if server.find(':') > -1: server, port = server.split(':', 1) else: port = '25' servers.append((server, int(port))) if options["smtpproxy", "listen_ports"]: splitPorts = options["smtpproxy", "listen_ports"] proxyPorts = map(_addressAndPort, splitPorts) if len(servers) != len(proxyPorts): print "smtpproxy:remote_servers & smtpproxy:listen_ports are " + \ "different lengths!" sys.exit() return servers, proxyPorts def CreateProxies(servers, proxyPorts, trainer): """Create BayesSMTPProxyListeners for all the given servers.""" # allow for old versions of pop3proxy if not isinstance(trainer, SMTPTrainer): trainer = SMTPTrainer(trainer.bayes, trainer) proxyListeners = [] for (server, serverPort), proxyPort in zip(servers, proxyPorts): listener = BayesSMTPProxyListener(server, serverPort, proxyPort, trainer) proxyListeners.append(listener) return proxyListeners def main(): """Runs the proxy until a 'KILL' command is received or someone hits Ctrl+Break.""" try: opts, args = getopt.getopt(sys.argv[1:], 'hvd:D:') except getopt.error, msg: print >>sys.stderr, str(msg) + '\n\n' + __doc__ sys.exit() bdbname = options["Storage", "persistent_storage_file"] useDBM = options["Storage", "persistent_use_database"] for opt, arg in opts: if opt == '-h': print >>sys.stderr, __doc__ sys.exit() elif opt == '-d': useDBM = False bdbname = arg elif opt == '-D': useDBM = True bdbname = arg elif opt == '-v': options["globals", "verbose"] = True bdbname = os.path.expanduser(bdbname) if options["globals", "verbose"]: print "Loading database %s..." % (bdbname), if useDBM: classifier = storage.DBDictClassifier(bdbname) else: classifier = storage.PickledClassifier(bdbname) if options["globals", "verbose"]: print "Done." servers, proxyPorts = LoadServerInfo() trainer = SMTPTrainer(classifier) proxyListeners = CreateProxies(servers, proxyPorts, trainer) Dibbler.run() if __name__ == '__main__': main() --- NEW FILE: sb_unheader.py --- #!/usr/bin/env python """ unheader.py: cleans headers from email messages. By default, this removes SpamAssassin headers, specify a pattern with -p to supply new headers to remove. This is often needed because existing spamassassin headers can provide killer spam clues, for all the wrong reasons. """ import re import sys import os import glob import mailbox import email.Parser import email.Message import email.Generator import getopt def unheader(msg, pat): pat = re.compile(pat) for hdr in msg.keys(): if pat.match(hdr): del msg[hdr] # remain compatible with 2.2.1 - steal replace_header from 2.3 source class Message(email.Message.Message): def replace_header(self, _name, _value): """Replace a header. Replace the first matching header found in the message, retaining header order and case. If no matching header was found, a KeyError is raised. """ _name = _name.lower() for i, (k, v) in zip(range(len(self._headers)), self._headers): if k.lower() == _name: self._headers[i] = (k, _value) break else: raise KeyError, _name class Parser(email.Parser.HeaderParser): def __init__(self): email.Parser.Parser.__init__(self, Message) def deSA(msg): if msg['X-Spam-Status']: if msg['X-Spam-Status'].startswith('Yes'): pct = msg['X-Spam-Prev-Content-Type'] if pct: msg['Content-Type'] = pct pcte = msg['X-Spam-Prev-Content-Transfer-Encoding'] if pcte: msg['Content-Transfer-Encoding'] = pcte subj = re.sub(r'\*\*\*\*\*SPAM\*\*\*\*\* ', '', msg['Subject'] or "") if subj != msg["Subject"]: msg.replace_header("Subject", subj) body = msg.get_payload() newbody = [] at_start = 1 for line in body.splitlines(): if at_start and line.startswith('SPAM: '): continue elif at_start: at_start = 0 newbody.append(line) msg.set_payload("\n".join(newbody)) unheader(msg, "X-Spam-") def process_message(msg, dosa, pats): if pats is not None: unheader(msg, pats) if dosa: deSA(msg) def process_mailbox(f, dosa=1, pats=None): gen = email.Generator.Generator(sys.stdout, maxheaderlen=0) for msg in mailbox.PortableUnixMailbox(f, Parser().parse): process_message(msg, dosa, pats) gen(msg, unixfrom=1) def process_maildir(d, dosa=1, pats=None): parser = Parser() for fn in glob.glob(os.path.join(d, "cur", "*")): print ("reading from %s..." % fn), file = open(fn) msg = parser.parse(file) process_message(msg, dosa, pats) tmpfn = os.path.join(d, "tmp", os.path.basename(fn)) tmpfile = open(tmpfn, "w") print "writing to %s" % tmpfn email.Generator.Generator(tmpfile, maxheaderlen=0)(msg, unixfrom=0) os.rename(tmpfn, fn) def usage(): print >> sys.stderr, "usage: unheader.py [ -p pat ... ] [ -s ] folder" print >> sys.stderr, "-p pat gives a regex pattern used to eliminate unwanted headers" print >> sys.stderr, "'-p pat' may be given multiple times" print >> sys.stderr, "-s tells not to remove SpamAssassin headers" print >> sys.stderr, "-d means treat folder as a Maildir" def main(args): headerpats = [] dosa = 1 ismbox = 1 try: opts, args = getopt.getopt(args, "p:shd") except getopt.GetoptError: usage() sys.exit(1) else: for opt, arg in opts: if opt == "-h": usage() sys.exit(0) elif opt == "-p": headerpats.append(arg) elif opt == "-s": dosa = 0 elif opt == "-d": ismbox = 0 pats = headerpats and "|".join(headerpats) or None if len(args) != 1: usage() sys.exit(1) if ismbox: f = file(args[0]) process_mailbox(f, dosa, pats) else: process_maildir(args[0], dosa, pats) if __name__ == "__main__": main(sys.argv[1:]) --- NEW FILE: sb_upload.py --- #!/usr/bin/env python """ Read a message or a mailbox file on standard input, upload it to a web browser and write it to standard output. usage: %(progname)s [-h] [-n] [-s server] [-p port] [-r N] Options: -h, --help - print help and exit -n, --null - suppress writing to standard output (default %(null)s) -s, --server= - provide alternate web server (default %(server)s) -p, --port= - provide alternate server port (default %(port)s) -r, --prob= - feed the message to the trainer w/ prob N [0.0...1.0] """ import sys import httplib import mimetypes import getopt import random from spambayes.Options import options progname = sys.argv[0] __author__ = "Skip Montanaro " __credits__ = "Spambayes gang, Wade Leftwich" try: True, False except NameError: # Maintain compatibility with Python 2.2 True, False = 1, 0 # appropriated verbatim from a recipe by Wade Leftwich in the Python # Cookbook: http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306 def post_multipart(host, selector, fields, files): """ Post fields and files to an http host as multipart/form-data. fields is a sequence of (name, value) elements for regular form fields. files is a sequence of (name, filename, value) elements for data to be uploaded as files. Return the server's response page. """ content_type, body = encode_multipart_formdata(fields, files) h = httplib.HTTP(host) h.putrequest('POST', selector) h.putheader('content-type', content_type) h.putheader('content-length', str(len(body))) h.endheaders() h.send(body) errcode, errmsg, headers = h.getreply() return h.file.read() def encode_multipart_formdata(fields, files): """ fields is a sequence of (name, value) elements for regular form fields. files is a sequence of (name, filename, value) elements for data to be uploaded as files. Return (content_type, body) ready for httplib.HTTP instance """ BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$' CRLF = '\r\n' L = [] for (key, value) in fields: L.append('--' + BOUNDARY) L.append('Content-Disposition: form-data; name="%s"' % key) L.append('') L.append(value) for (key, filename, value) in files: L.append('--' + BOUNDARY) L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename)) L.append('Content-Type: %s' % get_content_type(filename)) L.append('') L.append(value) L.append('--' + BOUNDARY + '--') L.append('') body = CRLF.join(L) content_type = 'multipart/form-data; boundary=%s' % BOUNDARY return content_type, body def get_content_type(filename): return mimetypes.guess_type(filename)[0] or 'application/octet-stream' def usage(*args): defaults = {} for d in args: defaults.update(d) print __doc__ % defaults def main(argv): null = False server = "localhost" port = options["html_ui", "port"] prob = 1.0 try: opts, args = getopt.getopt(argv, "hns:p:r:", ["help", "null", "server=", "port=", "prob="]) except getopt.error: usage(globals(), locals()) sys.exit(1) for opt, arg in opts: if opt in ("-h", "--help"): usage(globals(), locals()) sys.exit(0) elif opt in ("-n", "--null"): null = True elif opt in ("-s", "--server"): server = arg elif opt in ("-p", "--port"): port = int(arg) elif opt in ("-r", "--prob"): n = float(arg) if n < 0.0 or n > 1.0: usage(globals(), locals()) sys.exit(1) prob = n if args: usage(globals(), locals()) sys.exit(1) data = sys.stdin.read() sys.stdout.write(data) if random.random() < prob: try: post_multipart("%s:%d"%(server,port), "/upload", [], [('file', 'message.dat', data)]) except: # not an error if the server isn't responding pass if __name__ == "__main__": main(sys.argv[1:]) --- NEW FILE: sb_xmlrpcserver.py --- #! /usr/bin/env python # A server version of hammie.py """Usage: %(program)s [options] IP:PORT Where: -h show usage and exit -p FILE use file as the persistent store. loads data from this file if it exists, and saves data to this file at the end. Default: %(DEFAULTDB)s -d use the DBM store instead of cPickle. The file is larger and creating it is slower, but checking against it is much faster, especially for large word databases. IP IP address to bind (use 0.0.0.0 to listen on all IPs of this machine) PORT Port number to listen to. """ import SimpleXMLRPCServer import getopt import sys import traceback import xmlrpclib from spambayes import hammie try: True, False except NameError: # Maintain compatibility with Python 2.2 True, False = 1, 0 program = sys.argv[0] # For usage(); referenced by docstring above # Default DB path DEFAULTDB = hammie.DEFAULTDB class XMLHammie(hammie.Hammie): def score(self, msg, *extra): try: msg = msg.data except AttributeError: pass return xmlrpclib.Binary(hammie.Hammie.score(self, msg, *extra)) def filter(self, msg, *extra): try: msg = msg.data except AttributeError: pass return xmlrpclib.Binary(hammie.Hammie.filter(self, msg, *extra)) class HammieHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): def do_POST(self): """Handles the HTTP POST request. Attempts to interpret all HTTP POST requests as XML-RPC calls, which are forwarded to the _dispatch method for handling. This one also prints out tracebacks, to help me debug :) """ try: # get arguments data = self.rfile.read(int(self.headers["content-length"])) params, method = xmlrpclib.loads(data) # generate response try: response = self._dispatch(method, params) # wrap response in a singleton tuple response = (response,) except: traceback.print_exc() # report exception back to server response = xmlrpclib.dumps( xmlrpclib.Fault(1, "%s:%s" % (sys.exc_type, sys.exc_value)) ) else: response = xmlrpclib.dumps(response, methodresponse=1) except: # internal error, report as HTTP server error traceback.print_exc() print `data` self.send_response(500) self.end_headers() else: # got a valid XML RPC response self.send_response(200) self.send_header("Content-type", "text/xml") self.send_header("Content-length", str(len(response))) self.end_headers() self.wfile.write(response) # shut down the connection self.wfile.flush() self.connection.shutdown(1) def usage(code, msg=''): """Print usage message and sys.exit(code).""" if msg: print >> sys.stderr, msg print >> sys.stderr print >> sys.stderr, __doc__ % globals() sys.exit(code) def main(): """Main program; parse options and go.""" try: opts, args = getopt.getopt(sys.argv[1:], 'hdp:') except getopt.error, msg: usage(2, msg) pck = DEFAULTDB usedb = False for opt, arg in opts: if opt == '-h': usage(0) elif opt == '-p': pck = arg elif opt == "-d": usedb = True if len(args) != 1: usage(2, "IP:PORT not specified") ip, port = args[0].split(":") port = int(port) bayes = hammie.createbayes(pck, usedb) h = XMLHammie(bayes) server = SimpleXMLRPCServer.SimpleXMLRPCServer((ip, port), HammieHandler) server.register_instance(h) server.serve_forever() if __name__ == "__main__": main() --- sb-client.py DELETED --- --- sb-dbexpimp.py DELETED --- --- sb-filter.py DELETED --- --- sb-imapfilter.py DELETED --- --- sb-mailsort.py DELETED --- --- sb-mboxtrain.py DELETED --- --- sb-notesfilter.py DELETED --- --- sb-pop3dnd.py DELETED --- --- sb-server.py DELETED --- --- sb-smtpproxy.py DELETED --- --- sb-unheader.py DELETED --- --- sb-upload.py DELETED --- --- sb-xmlrpcserver.py DELETED --- From anadelonbrin at users.sourceforge.net Thu Sep 4 19:25:29 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Thu Sep 4 21:25:35 2003 Subject: [Spambayes-checkins] spambayes/spambayes Histogram.py, 1.2, 1.3 Tester.py, 1.2, 1.3 message.py, 1.34, 1.35 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv15577/spambayes Modified Files: Histogram.py Tester.py message.py Log Message: Missed some options cleanup. sb-server, sb-smtpproxy, sb-imapfilter, timtest and timcv all appear to work now. This should at least mean that the core does. Index: Histogram.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/Histogram.py,v retrieving revision 1.2 retrieving revision 1.3 diff -C2 -d -r1.2 -r1.3 *** Histogram.py 14 Jan 2003 05:38:20 -0000 1.2 --- Histogram.py 5 Sep 2003 01:25:27 -0000 1.3 *************** *** 18,22 **** # Note: nbuckets can be passed for backward compatibility. The # display() method can be passed a different nbuckets value. ! def __init__(self, nbuckets=options.nbuckets, lo=0.0, hi=100.0): self.lo, self.hi = lo, hi self.nbuckets = nbuckets --- 18,23 ---- # Note: nbuckets can be passed for backward compatibility. The # display() method can be passed a different nbuckets value. ! def __init__(self, nbuckets=options["TestDriver", "nbuckets"], ! lo=0.0, hi=100.0): self.lo, self.hi = lo, hi self.nbuckets = nbuckets *************** *** 76,80 **** # Compute percentiles. self.pct = pct = [] ! for p in options.percentiles: assert 0.0 <= p <= 100.0 # In going from data index 0 to index n-1, we move n-1 times. --- 77,81 ---- # Compute percentiles. self.pct = pct = [] ! for p in options["TestDriver", "percentiles"]: assert 0.0 <= p <= 100.0 # In going from data index 0 to index n-1, we move n-1 times. Index: Tester.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/Tester.py,v retrieving revision 1.2 retrieving revision 1.3 diff -C2 -d -r1.2 -r1.3 *** Tester.py 14 Jan 2003 05:38:20 -0000 1.2 --- Tester.py 5 Sep 2003 01:25:27 -0000 1.3 *************** *** 93,98 **** if callback: callback(example, prob) ! is_ham_guessed = prob < options.ham_cutoff ! is_spam_guessed = prob >= options.spam_cutoff if is_spam: self.nspam_tested += 1 --- 93,98 ---- if callback: callback(example, prob) ! is_ham_guessed = prob < options["Categorization", "ham_cutoff"] ! is_spam_guessed = prob >= options["Categorization", "spam_cutoff"] if is_spam: self.nspam_tested += 1 *************** *** 152,156 **** >>> from spambayes.classifier import Bayes >>> from spambayes.Options import options ! >>> options.ham_cutoff = options.spam_cutoff = 0.5 >>> good1 = _Example('', ['a', 'b', 'c']) --- 152,156 ---- >>> from spambayes.classifier import Bayes >>> from spambayes.Options import options ! >>> options["Categorization", "ham_cutoff"] = options["Categorization", "spam_cutoff"] = 0.5 >>> good1 = _Example('', ['a', 'b', 'c']) Index: message.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/message.py,v retrieving revision 1.34 retrieving revision 1.35 diff -C2 -d -r1.34 -r1.35 *** message.py 26 Aug 2003 06:49:56 -0000 1.34 --- message.py 5 Sep 2003 01:25:27 -0000 1.35 *************** *** 266,270 **** def setIdFromPayload(self): try: ! self.setId(self[options['pop3proxy','mailid_header_name']]) except ValueError: return None --- 266,270 ---- def setIdFromPayload(self): try: ! self.setId(self[options['Headers','mailid_header_name']]) except ValueError: return None *************** *** 336,340 **** if options['Headers','add_unique_id']: ! self[options['pop3proxy','mailid_header_name']] = self.id def delSBHeaders(self): --- 336,340 ---- if options['Headers','add_unique_id']: ! self[options['Headers','mailid_header_name']] = self.id def delSBHeaders(self): From anadelonbrin at users.sourceforge.net Thu Sep 4 19:25:29 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Thu Sep 4 21:25:43 2003 Subject: [Spambayes-checkins] spambayes/scripts sb_imapfilter.py,1.1,1.2 Message-ID: Update of /cvsroot/spambayes/spambayes/scripts In directory sc8-pr-cvs1:/tmp/cvs-serv15577/scripts Modified Files: sb_imapfilter.py Log Message: Missed some options cleanup. sb-server, sb-smtpproxy, sb-imapfilter, timtest and timcv all appear to work now. This should at least mean that the core does. Index: sb_imapfilter.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/scripts/sb_imapfilter.py,v retrieving revision 1.1 retrieving revision 1.2 diff -C2 -d -r1.1 -r1.2 *** sb_imapfilter.py 5 Sep 2003 01:16:45 -0000 1.1 --- sb_imapfilter.py 5 Sep 2003 01:25:27 -0000 1.2 *************** *** 581,588 **** cls = msg.GetClassification() ! if cls == options["Hammie", "header_ham_string"]: # we leave ham alone count["ham"] += 1 ! elif cls == options["Hammie", "header_spam_string"]: msg.MoveTo(spamfolder) count["spam"] += 1 --- 581,588 ---- cls = msg.GetClassification() ! if cls == options["Headers", "header_ham_string"]: # we leave ham alone count["ham"] += 1 ! elif cls == options["Headers", "header_spam_string"]: msg.MoveTo(spamfolder) count["spam"] += 1 From mhammond at users.sourceforge.net Thu Sep 4 20:09:44 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Thu Sep 4 22:09:55 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs __init__.py, 1.10, 1.11 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs In directory sc8-pr-cvs1:/tmp/cvs-serv22796 Modified Files: __init__.py Log Message: rc_size was not set when rc file did not exist. Index: __init__.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/__init__.py,v retrieving revision 1.10 retrieving revision 1.11 diff -C2 -d -r1.10 -r1.11 *** __init__.py 26 Aug 2003 21:29:08 -0000 1.10 --- __init__.py 5 Sep 2003 02:09:42 -0000 1.11 *************** *** 31,35 **** rc_size = stat_data[stat.ST_SIZE] except OSError: ! rc_mtime = None if rc_mtime!=mtime or rc_size!=size: # Need to generate the dialog. --- 31,35 ---- rc_size = stat_data[stat.ST_SIZE] except OSError: ! rc_mtime = rc_size = None if rc_mtime!=mtime or rc_size!=size: # Need to generate the dialog. From richiehindle at users.sourceforge.net Thu Sep 4 01:14:59 2003 From: richiehindle at users.sourceforge.net (Richie Hindle) Date: Thu Sep 4 22:40:49 2003 Subject: [Spambayes-checkins] spambayes WHAT_IS_NEW.txt,1.16,1.17 Message-ID: Update of /cvsroot/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv991 Modified Files: WHAT_IS_NEW.txt Log Message: Draw more attention to some incompatible options changes. Index: WHAT_IS_NEW.txt =================================================================== RCS file: /cvsroot/spambayes/spambayes/WHAT_IS_NEW.txt,v retrieving revision 1.16 retrieving revision 1.17 diff -C2 -d -r1.16 -r1.17 *** WHAT_IS_NEW.txt 2 Sep 2003 01:31:03 -0000 1.16 --- WHAT_IS_NEW.txt 4 Sep 2003 07:14:57 -0000 1.17 *************** *** 13,16 **** --- 13,30 ---- ====================== + -------------------------- + ** Incompatible changes ** + -------------------------- + + The values taken by some options have changed, so if you're upgrading from a + previous version, you may need to update your configuration file (.spambayesrc + or bayescustomize.ini) + + o allow_remote_connections now takes a list of allowed IP addresses, or the + word 'localhost', or an asterisk (meaning all connections are accepted). + o notate_to and notate_subject now take a comma-separated list of one or + more of 'spam', 'ham' and 'unsure', allowing you to control which classes + of message are notated. + Outlook Plugin -------------- From mhammond at users.sourceforge.net Fri Sep 5 00:51:03 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Fri Sep 5 02:51:07 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs/resources dialogs.rc, 1.36, 1.37 dialogs.h, 1.19, 1.20 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs/resources In directory sc8-pr-cvs1:/tmp/cvs-serv30645 Modified Files: dialogs.rc dialogs.h Log Message: Reference the folder bitmap in the resource file. Index: dialogs.rc =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/resources/dialogs.rc,v retrieving revision 1.36 retrieving revision 1.37 diff -C2 -d -r1.36 -r1.37 *** dialogs.rc 29 Aug 2003 06:23:51 -0000 1.36 --- dialogs.rc 5 Sep 2003 06:51:00 -0000 1.37 *************** *** 684,687 **** --- 684,694 ---- + ///////////////////////////////////////////////////////////////////////////// + // + // Bitmap + // + + IDB_FOLDERS BITMAP "folders.bmp" + #ifdef APSTUDIO_INVOKED ///////////////////////////////////////////////////////////////////////////// Index: dialogs.h =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/resources/dialogs.h,v retrieving revision 1.19 retrieving revision 1.20 diff -C2 -d -r1.19 -r1.20 *** dialogs.h 27 Aug 2003 00:07:07 -0000 1.19 --- dialogs.h 5 Sep 2003 06:51:00 -0000 1.20 *************** *** 9,13 **** #define IDD_FOLDER_SELECTOR 105 #define IDD_ADVANCED 106 - #define IDB_BITMAP1 107 #define IDD_GENERAL 108 #define IDD_FILTER_SPAM 110 --- 9,12 ---- *************** *** 26,29 **** --- 25,29 ---- #define IDD_WIZARD_FINISHED_TRAIN_LATER 124 #define IDB_SBWIZLOGO 125 + #define IDB_FOLDERS 127 #define IDC_PROGRESS 1000 #define IDC_PROGRESS_TEXT 1001 *************** *** 41,45 **** #define IDC_BUT_FILTER_ENABLE 1013 #define IDC_FILTER_STATUS 1014 - #define IDC_BUT_FILTER_NOW 1015 #define IDC_BUT_FILTER_DEFINE 1016 #define IDC_BUT_ABOUT 1017 --- 41,44 ---- *************** *** 48,56 **** #define IDC_BUT_UNREAD 1020 #define IDC_BUT_UNSEEN 1021 - #define IDC_BUT_FILTERNOW 1022 #define IDC_SLIDER_CERTAIN 1023 #define IDC_EDIT_CERTAIN 1024 #define IDC_ACTION_CERTAIN 1025 - #define IDC_TOFOLDER_CERTAIN 1026 #define IDC_FOLDER_CERTAIN 1027 #define IDC_BROWSE_CERTAIN 1028 --- 47,53 ---- *************** *** 58,62 **** #define IDC_EDIT_UNSURE 1030 #define IDC_ACTION_UNSURE 1031 - #define IDC_TOFOLDER_UNSURE 1032 #define IDC_FOLDER_UNSURE 1033 #define IDC_BROWSE_UNSURE 1034 --- 55,58 ---- *************** *** 71,80 **** #define IDC_STATUS1 1043 #define IDC_STATUS2 1044 - #define IDC_BUT_CLEARALL2 1045 #define IDC_BUT_NEW 1046 #define IDC_MARK_SPAM_AS_READ 1047 #define IDC_SAVE_SPAM_SCORE 1048 #define IDC_MARK_UNSURE_AS_READ 1051 - #define IDC_ADVANCED_BTN 1055 #define IDC_DELAY1_SLIDER 1056 #define IDC_DELAY1_TEXT 1057 --- 67,74 ---- *************** *** 85,90 **** #define IDB_SBLOGO 1062 #define IDC_LOGO_GRAPHIC 1063 - #define IDC_USE_DELAY1 1064 - #define IDC_USE_DELAY2 1065 #define IDC_TAB 1068 #define IDC_BACK_BTN 1069 --- 79,82 ---- *************** *** 95,102 **** #define IDC_DEL_SPAM_RS 1074 #define IDC_RECOVER_RS 1075 - #define IDC_HIDDEN 1076 #define IDC_FORWARD_BTN 1077 #define IDC_PAGE_PLACEHOLDER 1078 - #define IDC_SHOW_DATA_FOLDER2 1079 #define IDC_BUT_SHOW_DIAGNOSTICS 1080 #define IDC_BUT_PREPARATION 1081 --- 87,92 ---- *************** *** 112,116 **** #ifdef APSTUDIO_INVOKED #ifndef APSTUDIO_READONLY_SYMBOLS ! #define _APS_NEXT_RESOURCE_VALUE 126 #define _APS_NEXT_COMMAND_VALUE 40001 #define _APS_NEXT_CONTROL_VALUE 1094 --- 102,106 ---- #ifdef APSTUDIO_INVOKED #ifndef APSTUDIO_READONLY_SYMBOLS ! #define _APS_NEXT_RESOURCE_VALUE 128 #define _APS_NEXT_COMMAND_VALUE 40001 #define _APS_NEXT_CONTROL_VALUE 1094 From mhammond at users.sourceforge.net Fri Sep 5 00:53:26 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Fri Sep 5 02:53:30 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs/resources __init__.py, 1.2, 1.3 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs/resources In directory sc8-pr-cvs1:/tmp/cvs-serv30906 Modified Files: __init__.py Log Message: Add GetImageParamsFromBitmapID() to centralize bitmap location logic, hopefully taking into account a py2exe future. Index: __init__.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/resources/__init__.py,v retrieving revision 1.2 retrieving revision 1.3 diff -C2 -d -r1.2 -r1.3 *** __init__.py 10 Aug 2003 07:26:50 -0000 1.2 --- __init__.py 5 Sep 2003 06:53:24 -0000 1.3 *************** *** 1 **** ! # Empty file to designate this directory as a module. --- 1,22 ---- ! # Package that manages and defines dialog resources ! ! def GetImageParamsFromBitmapID(rc_parser, bmpid): ! import os, sys ! import win32gui, win32con, win32api ! if type(bmpid)==type(0): ! bmpid = rc_parser.names[bmpid] ! # For both binary and source versions, we currently load from files. ! # In future py2exe built binary versions we will be able to load the ! # bitmaps directly from our DLL. ! filename = rc_parser.bitmaps[bmpid] ! if hasattr(sys, "frozen"): ! # bitmap in the app/images directory ! # dont have manager available :( ! dll_filename = win32api.GetModuleFileName(sys.frozendllhandle) ! app_dir = os.path.dirname(dll_filename) ! filename = os.path.join(app_dir, "images", filename) ! else: ! if not os.path.isabs(filename): ! # In this directory ! filename = os.path.join( os.path.dirname( __file__ ), filename) ! return 0, filename, win32con.LR_LOADFROMFILE From mhammond at users.sourceforge.net Fri Sep 5 00:54:25 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Fri Sep 5 02:54:29 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs FolderSelector.py, 1.26, 1.27 processors.py, 1.11, 1.12 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs In directory sc8-pr-cvs1:/tmp/cvs-serv31065 Modified Files: FolderSelector.py processors.py Log Message: Centralize bitmap locating code (the folder selector didn't work in the binary build) Index: FolderSelector.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/FolderSelector.py,v retrieving revision 1.26 retrieving revision 1.27 diff -C2 -d -r1.26 -r1.27 *** FolderSelector.py 27 Aug 2003 06:53:27 -0000 1.26 --- FolderSelector.py 5 Sep 2003 06:54:23 -0000 1.27 *************** *** 426,437 **** win32gui.SendMessage(child, win32con.BM_SETCHECK, self.checkbox_state) self.list = self.GetDlgItem("IDC_LIST_FOLDERS") ! ! fname = os.path.join(os.path.dirname(__file__), "resources/folders.bmp") bitmapMask = win32api.RGB(0,0,255) ! self.imageList = win32gui.ImageList_LoadImage(0, fname, 16, 0, bitmapMask, win32con.IMAGE_BITMAP, ! win32con.LR_LOADFROMFILE) win32gui.SendMessage( self.list, commctrl.TVM_SETIMAGELIST, --- 426,438 ---- win32gui.SendMessage(child, win32con.BM_SETCHECK, self.checkbox_state) self.list = self.GetDlgItem("IDC_LIST_FOLDERS") ! import resources ! mod_handle, mod_bmp, extra_flags = \ ! resources.GetImageParamsFromBitmapID(self.dialog_parser, "IDB_FOLDERS") bitmapMask = win32api.RGB(0,0,255) ! self.imageList = win32gui.ImageList_LoadImage(mod_handle, mod_bmp, 16, 0, bitmapMask, win32con.IMAGE_BITMAP, ! extra_flags) win32gui.SendMessage( self.list, commctrl.TVM_SETIMAGELIST, *************** *** 466,470 **** timer.kill_timer(self.timer_id) self.item_map = None ! win32gui.ImageList_Destroy(self.imageList) FolderSelector_Parent.OnDestroy(self, hwnd, msg, wparam, lparam) --- 467,472 ---- timer.kill_timer(self.timer_id) self.item_map = None ! if self.imageList: ! win32gui.ImageList_Destroy(self.imageList) FolderSelector_Parent.OnDestroy(self, hwnd, msg, wparam, lparam) Index: processors.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/processors.py,v retrieving revision 1.11 retrieving revision 1.12 diff -C2 -d -r1.11 -r1.12 *** processors.py 27 Aug 2003 04:09:28 -0000 1.11 --- processors.py 5 Sep 2003 06:54:23 -0000 1.12 *************** *** 53,73 **** def Init(self): rcp = self.window.manager.dialog_parser; ! text = win32gui.GetWindowText(self.GetControl()) ! name = rcp.names[int(text)] ! if bitmap_handles.has_key(name): ! handle = bitmap_handles[name] else: ! filename = rcp.bitmaps[name] ! import os, sys ! if hasattr(sys, "frozen"): ! # bitmap in the app/images directory ! filename = os.path.join(self.window.manager.application_directory, ! "images", filename) ! else: ! if not os.path.isabs(filename): ! filename = os.path.join( os.path.dirname( __file__ ), "resources", filename) ! handle = win32gui.LoadImage(0, filename, win32con.IMAGE_BITMAP,0,0, ! win32con.LR_COLOR|win32con.LR_LOADFROMFILE|win32con.LR_SHARED) ! bitmap_handles[name] = handle win32gui.SendMessage(self.GetControl(), win32con.STM_SETIMAGE, win32con.IMAGE_BITMAP, handle) --- 53,67 ---- def Init(self): rcp = self.window.manager.dialog_parser; ! bmp_id = int(win32gui.GetWindowText(self.GetControl())) ! ! if bitmap_handles.has_key(bmp_id): ! handle = bitmap_handles[bmp_id] else: ! import resources ! mod_handle, mod_bmp, extra_flags = resources.GetImageParamsFromBitmapID(rcp, bmp_id) ! load_flags = extra_flags|win32con.LR_COLOR|win32con.LR_SHARED ! handle = win32gui.LoadImage(mod_handle, mod_bmp, ! win32con.IMAGE_BITMAP,0,0,load_flags) ! bitmap_handles[bmp_id] = handle win32gui.SendMessage(self.GetControl(), win32con.STM_SETIMAGE, win32con.IMAGE_BITMAP, handle) From richiehindle at users.sourceforge.net Fri Sep 5 01:06:52 2003 From: richiehindle at users.sourceforge.net (Richie Hindle) Date: Fri Sep 5 03:06:55 2003 Subject: [Spambayes-checkins] spambayes README-DEVEL.txt,1.1,1.2 Message-ID: Update of /cvsroot/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv551 Modified Files: README-DEVEL.txt Log Message: Added instructions on how to make a source release. Index: README-DEVEL.txt =================================================================== RCS file: /cvsroot/spambayes/spambayes/README-DEVEL.txt,v retrieving revision 1.1 retrieving revision 1.2 diff -C2 -d -r1.1 -r1.2 *** README-DEVEL.txt 25 Aug 2003 01:36:41 -0000 1.1 --- README-DEVEL.txt 5 Sep 2003 07:06:50 -0000 1.2 *************** *** 319,320 **** --- 319,353 ---- factor of 10 or more across training sets, and even within runs against a single training set), but harder to spot using N-fold c-v. + + + Making a source release + ======================= + + Source releases are built with distutils. Here's how I (Richie) have been + building them. I do this on a Windows box, partly so that the zip release + can have Windows line endings without needing to run a conversion script. + I don't think that's actually necessary, because everything would work on + Windows even with Unix line endings, but you couldn't load the files into + Notepad and sometimes it's convenient to do so. End users might not even + have any other text editor, so it make things like the README unREADable. + 8-) + + o Checkout the 'spambayes' module twice, once with Windows line endings + and once with Unix line endings (I use WinCVS for this, using "Admin / + Preferences / Globals / Checkout text files with the Unix LF". + o Change spambayes/__init__.py to contain the new version number. + o In the Windows checkout, run "python setup.py sdist --formats zip" + o In the Unix checkout, run "python setup.py sdist --formats gztar" + o Take the resulting spambayes-1.0a5.zip and spambayes-1.0a5.tar.gz, and + test the former on Windows (ideally in a freshly-installed Python + environment; I keep a VMWare snapshot of a clean Windows installation + for this, but that's probably overkill 8-) and test the latter on Unix + (a Debian VMWare box in my case). + o Dance the SourceForge release dance: + http://sourceforge.net/docman/display_doc.php?docid=6445&group_id=1#filereleasesteps + When it comes to the "what's new" and the ChangeLog, I cut'n'paste the + relevant pieces of WHAT_IS_NEW.txt and CHANGELOG.txt into the form, and + check the "Keep my preformatted text" checkbox. + + The announce the release on spambayes-announce, spambayes and spambayes-dev + (why do things by halves?) and whatch the bug reports roll in. 8-) From richiehindle at users.sourceforge.net Fri Sep 5 01:16:38 2003 From: richiehindle at users.sourceforge.net (Richie Hindle) Date: Fri Sep 5 03:16:41 2003 Subject: [Spambayes-checkins] spambayes README-DEVEL.txt,1.2,1.3 Message-ID: Update of /cvsroot/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv2200 Modified Files: README-DEVEL.txt Log Message: Today's Top Tip: It helps if you hit the Save button before committing. Index: README-DEVEL.txt =================================================================== RCS file: /cvsroot/spambayes/spambayes/README-DEVEL.txt,v retrieving revision 1.2 retrieving revision 1.3 diff -C2 -d -r1.2 -r1.3 *** README-DEVEL.txt 5 Sep 2003 07:06:50 -0000 1.2 --- README-DEVEL.txt 5 Sep 2003 07:16:36 -0000 1.3 *************** *** 333,340 **** 8-) o Checkout the 'spambayes' module twice, once with Windows line endings and once with Unix line endings (I use WinCVS for this, using "Admin / Preferences / Globals / Checkout text files with the Unix LF". ! o Change spambayes/__init__.py to contain the new version number. o In the Windows checkout, run "python setup.py sdist --formats zip" o In the Unix checkout, run "python setup.py sdist --formats gztar" --- 333,346 ---- 8-) + o If any new file types have been added since last time (eg. 1.0a5 went + out without the Windows .rc and .h files) then add them to MANIFEST.in. + If there are any new scripts or packages, add them to setup.py. Test + these changes (by building source packages according to the instructions + below) then commit your edits. o Checkout the 'spambayes' module twice, once with Windows line endings and once with Unix line endings (I use WinCVS for this, using "Admin / Preferences / Globals / Checkout text files with the Unix LF". ! o Change spambayes/__init__.py to contain the new version number but don't ! commit it yet, just in case something goes wrong. o In the Windows checkout, run "python setup.py sdist --formats zip" o In the Unix checkout, run "python setup.py sdist --formats gztar" *************** *** 349,353 **** relevant pieces of WHAT_IS_NEW.txt and CHANGELOG.txt into the form, and check the "Keep my preformatted text" checkbox. ! The announce the release on spambayes-announce, spambayes and spambayes-dev ! (why do things by halves?) and whatch the bug reports roll in. 8-) --- 355,362 ---- relevant pieces of WHAT_IS_NEW.txt and CHANGELOG.txt into the form, and check the "Keep my preformatted text" checkbox. + o Now commit spambayes/__init__.py and tag the whole checkout - see the + existing tag names for the tag name format. + o Update the website News section. ! Then announce the release on the mailing lists and watch the bug reports ! roll in. 8-) From mhammond at users.sourceforge.net Fri Sep 5 01:21:16 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Fri Sep 5 03:21:26 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000 msgstore.py, 1.72, 1.73 tester.py, 1.17, 1.18 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000 In directory sc8-pr-cvs1:/tmp/cvs-serv3001 Modified Files: msgstore.py tester.py Log Message: Guard against invalid folder IDs better, and beef up the test suite to check this and a few other things. Index: msgstore.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/msgstore.py,v retrieving revision 1.72 retrieving revision 1.73 diff -C2 -d -r1.72 -r1.73 *** msgstore.py 4 Sep 2003 12:14:12 -0000 1.72 --- msgstore.py 5 Sep 2003 07:21:14 -0000 1.73 *************** *** 57,64 **** def __str__(self): try: ! return "%s: %s" % (self.__class__.__name__, ! GetCOMExceptionString(self.mapi_exception)) except: ! print "Error __str__" import traceback traceback.print_exc() --- 57,69 ---- def __str__(self): try: ! if self.mapi_exception is not None: ! err_str = GetCOMExceptionString(self.mapi_exception) ! else: ! err_str = self.extra_msg or '' ! return "%s: %s" % (self.__class__.__name__, err_str) ! # Python silently consumes exceptions here, and uses ! # except: ! print "FAILED to str() a MsgStore exception!" import traceback traceback.print_exc() *************** *** 244,249 **** assert type(item_id)==type(()), \ "Item IDs must be a tuple (not a %r)" % item_id ! store_id, entry_id = item_id ! return mapi.BinFromHex(store_id), mapi.BinFromHex(entry_id) def _GetSubFolderIter(self, folder): --- 249,257 ---- assert type(item_id)==type(()), \ "Item IDs must be a tuple (not a %r)" % item_id ! try: ! store_id, entry_id = item_id ! return mapi.BinFromHex(store_id), mapi.BinFromHex(entry_id) ! except ValueError: ! raise MsgStoreException(None, "The specified ID '%s' is invalid" % (item_id,)) def _GetSubFolderIter(self, folder): *************** *** 264,268 **** def GetFolderGenerator(self, folder_ids, include_sub): for folder_id in folder_ids: ! folder_id = self.NormalizeID(folder_id) try: folder = self._OpenEntry(folder_id) --- 272,280 ---- def GetFolderGenerator(self, folder_ids, include_sub): for folder_id in folder_ids: ! try: ! folder_id = self.NormalizeID(folder_id) ! except MsgStoreException, details: ! print "NOTE: Skipping invalid folder", details ! continue try: folder = self._OpenEntry(folder_id) Index: tester.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/tester.py,v retrieving revision 1.17 retrieving revision 1.18 diff -C2 -d -r1.17 -r1.18 *** tester.py 4 Sep 2003 12:14:12 -0000 1.17 --- tester.py 5 Sep 2003 07:21:14 -0000 1.18 *************** *** 12,15 **** --- 12,16 ---- from win32com.client import constants + import sys from time import sleep import copy *************** *** 18,21 **** --- 19,24 ---- import threading + import msgstore + from win32com.mapi import mapi, mapiutil import pythoncom *************** *** 33,36 **** --- 36,50 ---- raise TestFailure(msg) + def AssertRaises(exception, func, *args): + try: + func(*args) + raise TestFailed("Function '%s' should have raised '%r', but it worked!" % \ + (func, exception)) + except: + exc_type = sys.exc_info()[0] + if exc_type == exception or issubclass(exc_type, exception): + return + raise + filter_event = threading.Event() *************** *** 352,366 **** def TestHamFilter(driver): ! _DoTestHamFilter(driver, driver.folder_watch) ! # Try again, with the secondary folder if it exists (it is likely one ! # is the inbox and one isn't, so may be useful) ! if driver.folder_watch_2 is not None: ! _DoTestHamFilter(driver, driver.folder_watch_2) ! # Now test our incremental train logic ! _DoTestHamTrain(driver, driver.folder_watch, driver.folder_watch_2) ! else: ! print "Skipping testing secondary watch folder filtering - " \ ! "only one watch folder is configured" ! print "Created a Ham message, and saw it remain in place." def TestUnsureFilter(driver): --- 366,381 ---- def TestHamFilter(driver): ! # Execute the 'ham' test in every folder we watch. ! mgr = driver.manager ! gen = mgr.message_store.GetFolderGenerator( ! mgr.config.filter.watch_folder_ids, ! mgr.config.filter.watch_include_sub) ! num = 0 ! for f in gen: ! print "Running ham filter tests on folder '%s'" % f.GetFQName() ! f = f.GetOutlookItem() ! _DoTestHamFilter(driver, f) ! num += 1 ! print "Created a Ham message, and saw it remain in place (in %d watch folders.)" % num def TestUnsureFilter(driver): *************** *** 442,446 **** # Check messages we are unable to score. # Must enable the filtering code for this test - import msgstore msgstore.test_suite_running = False try: --- 457,460 ---- *************** *** 482,485 **** --- 496,524 ---- msgstore.test_suite_running = True + def run_invalid_id_tests(manager): + # Do some tests with invalid message and folder IDs. + print "Doing some 'invalid ID' tests - you should see a couple of warning, but no errors or tracebacks" + id_no_item = ('0000','0000') # this ID is 'valid' - but there will be no such item + id_invalid = ('xxxx','xxxx') # this ID is 'invalid' in that the hex-bin conversion fails + id_empty1 = ('','') + id_empty2 = () + bad_ids = id_no_item, id_invalid, id_empty1, id_empty2 + for id in bad_ids: + AssertRaises(msgstore.MsgStoreException, manager.message_store.GetMessage, id) + # Test 'GetFolderGenerator' works with invalid ids. + for id in bad_ids: + AssertRaises(msgstore.MsgStoreException, manager.message_store.GetFolder, id) + ids = manager.config.filter.watch_folder_ids[:] + ids.append(id) + found = 0 + for f in manager.message_store.GetFolderGenerator(ids, False): + found += 1 + if found > len(manager.config.filter.watch_folder_ids): + raise TestFailed("Seemed to find the extra folder") + names = manager.FormatFolderNames(ids, False) + if names.find(" Update of /cvsroot/spambayes/website In directory sc8-pr-cvs1:/tmp/cvs-serv10448 Modified Files: applications.ht download.ht Log Message: More references to 1.0a4. I've left the Outlook plug-in source instructions to get 1.0a4 since 1.0a5 is missing files. Index: applications.ht =================================================================== RCS file: /cvsroot/spambayes/website/applications.ht,v retrieving revision 1.20 retrieving revision 1.21 diff -C2 -d -r1.20 -r1.21 *** applications.ht 20 Aug 2003 04:26:33 -0000 1.20 --- applications.ht 5 Sep 2003 08:02:54 -0000 1.21 *************** *** 33,37 ****

Alternatively, you can use CVS to get the code - go to the CVS page on the project's sourceforge site for more.

!

hammie.py

hammie is a command line tool for marking mail as ham or spam. Skip Montanaro has started a guide to integrating hammie with your mailer (Unix-only instructions at the moment - additions welcome!). --- 33,37 ----

Alternatively, you can use CVS to get the code - go to the CVS page on the project's sourceforge site for more.

!

hammiefilter.py

hammie is a command line tool for marking mail as ham or spam. Skip Montanaro has started a guide to integrating hammie with your mailer (Unix-only instructions at the moment - additions welcome!). *************** *** 45,49 ****

Availability

!

Download the alpha4 release.

Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.

--- 45,49 ----

Availability

!

Download the alpha5 release.

Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.

*************** *** 77,81 ****

Availability

!

Download the alpha4 release.

Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.

--- 77,81 ----

Availability

!

Download the alpha5 release.

Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.

*************** *** 93,97 ****

Availability

!

Download the alpha4 release.

Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.

--- 93,97 ----

Availability

!

Download the alpha5 release.

Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.

*************** *** 111,114 ****

Availability

!

Download the alpha4 release.

Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.

--- 111,114 ----

Availability

!

Download the alpha5 release.

Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.

Index: download.ht =================================================================== RCS file: /cvsroot/spambayes/website/download.ht,v retrieving revision 1.16 retrieving revision 1.17 diff -C2 -d -r1.16 -r1.17 *** download.ht 2 Sep 2003 03:41:55 -0000 1.16 --- download.ht 5 Sep 2003 08:02:54 -0000 1.17 *************** *** 4,9 ****

Source Releases

!

The fourth pre-release of version 1.0 of the SpamBayes project is available. ! Download version 1.0a4 from the sourceforge Files page as either a gzipped tarball or a zip file of the source files.

--- 4,9 ----

Source Releases

!

The fifth pre-release of version 1.0 of the SpamBayes project is available. ! Download version 1.0a5 from the sourceforge Files page as either a gzipped tarball or a zip file of the source files.

*************** *** 34,38 **** see here for more details. Note that due to capacity problems with Sourceforge, ! the public CVS servers often run up to 24 hours behind the real CVS servers. This is something that SF are working on improving.

--- 34,38 ---- see here for more details. Note that due to capacity problems with Sourceforge, ! the public CVS servers often run up to 48 hours behind the real CVS servers. This is something that SF are working on improving.

From anadelonbrin at users.sourceforge.net Fri Sep 5 02:04:54 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Fri Sep 5 04:04:58 2003 Subject: [Spambayes-checkins] spambayes README-DEVEL.txt,1.3,1.4 Message-ID: Update of /cvsroot/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv10746 Modified Files: README-DEVEL.txt Log Message: Add references to the other pages that the version number is referenced on. Index: README-DEVEL.txt =================================================================== RCS file: /cvsroot/spambayes/spambayes/README-DEVEL.txt,v retrieving revision 1.3 retrieving revision 1.4 diff -C2 -d -r1.3 -r1.4 *** README-DEVEL.txt 5 Sep 2003 07:16:36 -0000 1.3 --- README-DEVEL.txt 5 Sep 2003 08:04:51 -0000 1.4 *************** *** 357,361 **** o Now commit spambayes/__init__.py and tag the whole checkout - see the existing tag names for the tag name format. ! o Update the website News section. Then announce the release on the mailing lists and watch the bug reports --- 357,361 ---- o Now commit spambayes/__init__.py and tag the whole checkout - see the existing tag names for the tag name format. ! o Update the website News, Download and Application sections. Then announce the release on the mailing lists and watch the bug reports From anadelonbrin at users.sourceforge.net Fri Sep 5 03:21:04 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Fri Sep 5 05:21:09 2003 Subject: [Spambayes-checkins] website faq.txt,1.39,1.40 Message-ID: Update of /cvsroot/spambayes/website In directory sc8-pr-cvs1:/tmp/cvs-serv22688 Modified Files: faq.txt Log Message: Add a FAQ about the 'access denied' problem that a lot of pop3proxy people seem to run into. We might want to special case this error and tell them what to do instead, or maybe even just automatically try a range of different ports. Index: faq.txt =================================================================== RCS file: /cvsroot/spambayes/website/faq.txt,v retrieving revision 1.39 retrieving revision 1.40 diff -C2 -d -r1.39 -r1.40 *** faq.txt 27 Aug 2003 03:53:03 -0000 1.39 --- faq.txt 5 Sep 2003 09:21:02 -0000 1.40 *************** *** 933,936 **** --- 933,947 ---- + I've just installed SpamBayes, but when I run it I get an access denied error. + ------------------------------------------------------------------------------ + + If you use pop3proxy or imapfilter, haven't set anything up, and run it and + get an error that ends with ``socket.error: (10013, 'Permission denied')``, + then this probably means that port 8880, which SpamBayes is trying to use to + present the web interface, is already taken on your machine. Try using + ``pop3proxy.py -u 8881 -b`` (or ``imapfilter.py -u 8881 -b``), or another + port that you know is free and available on your machine. + + Known Problems & Workarounds ============================ From mhammond at users.sourceforge.net Fri Sep 5 05:49:49 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Fri Sep 5 07:49:54 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000/docs configuration.html, 1.8, 1.9 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000/docs In directory sc8-pr-cvs1:/tmp/cvs-serv14014/docs Modified Files: configuration.html Log Message: Clarify rules for "data_directory" based on bug report. Index: configuration.html =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/docs/configuration.html,v retrieving revision 1.8 retrieving revision 1.9 diff -C2 -d -r1.8 -r1.9 *** configuration.html 26 Aug 2003 02:35:45 -0000 1.8 --- configuration.html 5 Sep 2003 11:49:47 -0000 1.9 *************** *** 186,190 **** The directory where SpamBayes ! will save its database and primary configuration file.  This can only be set in the 'early' configuration files - see below.
--- 186,193 ---- The directory where SpamBayes ! will save its database and primary configuration file.  This ! directory name should not be enclosed in quotes, but can include spaces ! and extended characters (in which case the default windows encoding is ! assumed).  This can only be set in the 'early' configuration files - see below.
From mhammond at users.sourceforge.net Fri Sep 5 05:50:59 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Fri Sep 5 07:51:03 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs dialog_map.py, 1.32, 1.33 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs In directory sc8-pr-cvs1:/tmp/cvs-serv14164/dialogs Modified Files: dialog_map.py Log Message: Claify wording when untrained, so it doesn't sound like 'unsure' is a folder name. Index: dialog_map.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/dialog_map.py,v retrieving revision 1.32 retrieving revision 1.33 diff -C2 -d -r1.32 -r1.33 *** dialog_map.py 29 Aug 2003 06:23:50 -0000 1.32 --- dialog_map.py 5 Sep 2003 11:50:57 -0000 1.33 *************** *** 42,47 **** else: db_status = "Database has no training information. SpamBayes " \ ! "will deliver all messages to your 'Unsure' folder, " \ ! "ready for you to classify." win32gui.SendMessage(self.GetControl(), win32con.WM_SETTEXT, 0, db_status) --- 42,47 ---- else: db_status = "Database has no training information. SpamBayes " \ ! "will classify all messages as 'unsure', " \ ! "ready for you to train." win32gui.SendMessage(self.GetControl(), win32con.WM_SETTEXT, 0, db_status) From mhammond at users.sourceforge.net Fri Sep 5 05:54:21 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Fri Sep 5 07:54:26 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000 manager.py, 1.84, 1.85 msgstore.py, 1.73, 1.74 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000 In directory sc8-pr-cvs1:/tmp/cvs-serv14453 Modified Files: manager.py msgstore.py Log Message: Exception handling functions also decode Outlook exceptions (which are 'extended' exceptions, with the underlying MAPI code buried). When we fail to add the 'Spam' field to a read-only store (eg, hotmail), complain less loudly. Index: manager.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/manager.py,v retrieving revision 1.84 retrieving revision 1.85 diff -C2 -d -r1.84 -r1.85 *** manager.py 4 Sep 2003 12:14:12 -0000 1.84 --- manager.py 5 Sep 2003 11:54:19 -0000 1.85 *************** *** 551,556 **** folder = msgstore_folder.GetOutlookItem() self.LogDebug(2, "Checking folder '%s' for field '%s'" \ ! % (folder.Name.encode("mbcs", "replace"), self.config.general.field_score_name)) items = folder.Items item = items.GetFirst() --- 551,557 ---- folder = msgstore_folder.GetOutlookItem() + folder_name = msgstore_folder.GetFQName() self.LogDebug(2, "Checking folder '%s' for field '%s'" \ ! % (folder_name, self.config.general.field_score_name)) items = folder.Items item = items.GetFirst() *************** *** 581,588 **** self.LogDebug(2, "Created the UserProperty!") except pythoncom.com_error, details: ! print "Warning: failed to create the Outlook " \ ! "user-property in folder '%s'" \ ! % (folder.Name.encode("mbcs", "replace"),) ! print "", details if include_sub: # Recurse down the folder list. --- 582,593 ---- self.LogDebug(2, "Created the UserProperty!") except pythoncom.com_error, details: ! if msgstore.IsReadOnlyCOMException(details): ! self.LogDebug(1, "The folder '%s' is read-only - user property can't be added" % \ ! (folder_name,)) ! else: ! print "Warning: failed to create the Outlook " \ ! "user-property in folder '%s'" \ ! % (folder_name,) ! print "", details if include_sub: # Recurse down the folder list. Index: msgstore.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/msgstore.py,v retrieving revision 1.73 retrieving revision 1.74 diff -C2 -d -r1.73 -r1.74 *** msgstore.py 5 Sep 2003 07:21:14 -0000 1.73 --- msgstore.py 5 Sep 2003 11:54:19 -0000 1.74 *************** *** 15,18 **** --- 15,19 ---- from win32com.mapi.mapitags import * import pythoncom + import winerror # Additional MAPI constants we dont have in Python *************** *** 83,86 **** --- 84,91 ---- pass + # The object has changed since it was opened. + class ObjectChangedException(MsgStoreException): + pass + # Utility functions for exceptions. Convert a COM exception to the best # manager exception. *************** *** 90,98 **** if IsReadOnlyCOMException(com_exc): return ReadOnlyException(com_exc) return MsgStoreException(com_exc) # Build a reasonable string from a COM exception tuple def GetCOMExceptionString(exc_val): ! hr, msg, exc, arg_err = exc_val err_string = mapiutil.GetScodeString(hr) return "Exception 0x%x (%s): %s" % (hr, err_string, msg) --- 95,114 ---- if IsReadOnlyCOMException(com_exc): return ReadOnlyException(com_exc) + scode = NormalizeCOMException(com_exc)[0] + # And simple scode based ones. + if scode == mapi.MAPI_E_OBJECT_CHANGED: + return ObjectChangedException(com_exc) return MsgStoreException(com_exc) + def NormalizeCOMException(exc_val): + hr, msg, exc, arg_err = exc_val + if hr == winerror.DISP_E_EXCEPTION and exc: + # 'client' exception - unpack 'exception object' + wcode, source, msg, help1, help2, hr = exc + return hr, msg, exc, arg_err + # Build a reasonable string from a COM exception tuple def GetCOMExceptionString(exc_val): ! hr, msg, exc, arg_err = NormalizeCOMException(exc_val) err_string = mapiutil.GetScodeString(hr) return "Exception 0x%x (%s): %s" % (hr, err_string, msg) *************** *** 100,104 **** # Does this exception probably mean "object not found"? def IsNotFoundCOMException(exc_val): ! hr, msg, exc, arg_err = exc_val return hr in [mapi.MAPI_E_OBJECT_DELETED, mapi.MAPI_E_NOT_FOUND] --- 116,120 ---- # Does this exception probably mean "object not found"? def IsNotFoundCOMException(exc_val): ! hr, msg, exc, arg_err = NormalizeCOMException(exc_val) return hr in [mapi.MAPI_E_OBJECT_DELETED, mapi.MAPI_E_NOT_FOUND] *************** *** 106,110 **** # in, or 'cos the server is down"? def IsNotAvailableCOMException(exc_val): ! hr, msg, exc, arg_err = exc_val return hr == mapi.MAPI_E_FAILONEPROVIDER --- 122,126 ---- # in, or 'cos the server is down"? def IsNotAvailableCOMException(exc_val): ! hr, msg, exc, arg_err = NormalizeCOMException(exc_val) return hr == mapi.MAPI_E_FAILONEPROVIDER *************** *** 113,118 **** # and also for hotmail messages (0x8004dff7) known_failure_codes = -2146644781, -2147164169 return exc_val[0] in known_failure_codes - def ReportMAPIError(manager, what, exc_val): --- 129,134 ---- # and also for hotmail messages (0x8004dff7) known_failure_codes = -2146644781, -2147164169 + exc_val = NormalizeCOMException(exc_val) return exc_val[0] in known_failure_codes def ReportMAPIError(manager, what, exc_val): From montanaro at users.sourceforge.net Fri Sep 5 09:18:06 2003 From: montanaro at users.sourceforge.net (Skip Montanaro) Date: Fri Sep 5 11:18:11 2003 Subject: [Spambayes-checkins] spambayes/spambayes hammie.py,1.8,1.9 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv23782 Modified Files: hammie.py Log Message: hammie_train_on_filter -> train_on_filter - option change was missed Index: hammie.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/hammie.py,v retrieving revision 1.8 retrieving revision 1.9 diff -C2 -d -r1.8 -r1.9 *** hammie.py 5 Sep 2003 01:15:28 -0000 1.8 --- hammie.py 5 Sep 2003 15:18:03 -0000 1.9 *************** *** 99,103 **** debug = options["Hammie", "debug_header"] if train == None: ! train = options["Hammie", "hammie_train_on_filter"] msg = mboxutils.get_message(msg) --- 99,103 ---- debug = options["Hammie", "debug_header"] if train == None: ! train = options["Hammie", "train_on_filter"] msg = mboxutils.get_message(msg) From montanaro at users.sourceforge.net Fri Sep 5 09:25:38 2003 From: montanaro at users.sourceforge.net (Skip Montanaro) Date: Fri Sep 5 11:25:42 2003 Subject: [Spambayes-checkins] spambayes/scripts sb_chkopts.py,NONE,1.1 Message-ID: Update of /cvsroot/spambayes/spambayes/scripts In directory sc8-pr-cvs1:/tmp/cvs-serv25157 Added Files: sb_chkopts.py Log Message: trivial options checker --- NEW FILE: sb_chkopts.py --- #!/usr/bin/env python """ Trivial script to check that the user's options file doesn't contain any old-style option names. """ from spambayes import Options From anadelonbrin at users.sourceforge.net Fri Sep 5 21:12:49 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Fri Sep 5 23:12:54 2003 Subject: [Spambayes-checkins] spambayes MANIFEST.in,1.5,1.6 Message-ID: Update of /cvsroot/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv27451 Modified Files: MANIFEST.in Log Message: Fix [ 800555 ] 1.0a5 release missing key outlook files. Index: MANIFEST.in =================================================================== RCS file: /cvsroot/spambayes/spambayes/MANIFEST.in,v retrieving revision 1.5 retrieving revision 1.6 diff -C2 -d -r1.5 -r1.6 *** MANIFEST.in 7 Jul 2003 00:09:27 -0000 1.5 --- MANIFEST.in 6 Sep 2003 03:12:46 -0000 1.6 *************** *** 3,7 **** recursive-include pspam *.py *.txt *.ini *.sh recursive-include contrib *.py *.sh *.txt *.el *rc ! recursive-include Outlook2000 *.py *.txt *.ini *.html *.bmp recursive-include utilities *.py *.txt recursive-include testtools *.py *.txt --- 3,7 ---- recursive-include pspam *.py *.txt *.ini *.sh recursive-include contrib *.py *.sh *.txt *.el *rc ! recursive-include Outlook2000 *.py *.txt *.ini *.html *.bmp *.rc *.h recursive-include utilities *.py *.txt recursive-include testtools *.py *.txt From anadelonbrin at users.sourceforge.net Fri Sep 5 22:03:37 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Sat Sep 6 00:03:40 2003 Subject: [Spambayes-checkins] spambayes/scripts sb_pop3dnd.py,1.1,1.2 Message-ID: Update of /cvsroot/spambayes/spambayes/scripts In directory sc8-pr-cvs1:/tmp/cvs-serv1864/scripts Modified Files: sb_pop3dnd.py Log Message: Opps. I should have done this before. Add a missing file that's necessary to try out the sb_pop3dnd.py script, and the version and options necessary. Also update to reflect the pop3proxy.py renaming. Index: sb_pop3dnd.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/scripts/sb_pop3dnd.py,v retrieving revision 1.1 retrieving revision 1.2 diff -C2 -d -r1.1 -r1.2 *** sb_pop3dnd.py 5 Sep 2003 01:16:45 -0000 1.1 --- sb_pop3dnd.py 6 Sep 2003 04:03:34 -0000 1.2 *************** *** 3,8 **** from __future__ import generators ! """ ! Overkill (someone *please* come up with something to call this script!) This application is a twisted cross between a POP3 proxy and an IMAP --- 3,7 ---- from __future__ import generators ! """POP3DND - provides drag'n'drop training ability for POP3 clients. This application is a twisted cross between a POP3 proxy and an IMAP *************** *** 133,137 **** from spambayes.ServerUI import ServerUserInterface from spambayes.UserInterface import UserInterfaceServer ! from pop3proxy import POP3ProxyBase, State, _addressPortStr, _recreateState def ensureDir(dirname): --- 132,136 ---- from spambayes.ServerUI import ServerUserInterface from spambayes.UserInterface import UserInterfaceServer ! from sb_server import POP3ProxyBase, State, _addressPortStr, _recreateState def ensureDir(dirname): *************** *** 928,932 **** # Let the user know what they are using... print get_version_string("IMAP Server") ! print "and engine %s.\n" % (get_version_string(),) # setup everything --- 927,934 ---- # Let the user know what they are using... print get_version_string("IMAP Server") ! print get_version_string("POP3 Proxy") ! print "and engine %s," % (get_version_string(),) ! from twisted.copyright import version as twisted_version ! print "with twisted version %s.\n" % (twisted_version,) # setup everything From anadelonbrin at users.sourceforge.net Fri Sep 5 22:03:37 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Sat Sep 6 00:03:43 2003 Subject: [Spambayes-checkins] spambayes/spambayes ServerUI.py, NONE, 1.1 Options.py, 1.70, 1.71 Version.py, 1.17, 1.18 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv1864/spambayes Modified Files: Options.py Version.py Added Files: ServerUI.py Log Message: Opps. I should have done this before. Add a missing file that's necessary to try out the sb_pop3dnd.py script, and the version and options necessary. Also update to reflect the pop3proxy.py renaming. --- NEW FILE: ServerUI.py --- """IMAP Server Web Interface Classes: ServerInterface - Interface class for imapserver Abstract: This module implements a browser based Spambayes user interface for the IMAP server. Users may use it to interface with the server. The following functions are currently included: [From the base class UserInterface] onClassify - classify a given message onWordquery - query a word from the database onTrain - train a message or mbox onSave - save the database and possibly shutdown [Here] onHome - a home page with various options To do: o Suggestions? """ # This module is part of the spambayes project, which is Copyright 2002-3 # The Python Software Foundation and is covered by the Python Software # Foundation license. __author__ = "Tony Meyer " __credits__ = "All the Spambayes folk." try: True, False except NameError: # Maintain compatibility with Python 2.2 True, False = 1, 0 from spambayes import UserInterface from spambayes.Options import options # These are the options that will be offered on the configuration page. # If the option is None, then the entry is a header and the following # options will appear in a new box on the configuration page. # These are also used to generate http request parameters and template # fields/variables. parm_ini_map = ( ('POP3 Proxy Options', None), ('pop3proxy', 'remote_servers'), ('pop3proxy', 'listen_ports'), ('IMAP Server Options', None), ('imapserver', 'username'), ('imapserver', 'password'), ('imapserver', 'port'), ('Storage Options', None), ('Storage', 'persistent_storage_file'), ('Storage', 'messageinfo_storage_file'), ('Statistics Options', None), ('Categorization', 'ham_cutoff'), ('Categorization', 'spam_cutoff'), ('Classifier', 'experimental_ham_spam_imbalance_adjustment'), ) class ServerUserInterface(UserInterface.UserInterface): """Serves the HTML user interface for the server.""" def __init__(self, state, state_recreator): UserInterface.UserInterface.__init__(self, state.bayes, parm_ini_map) self.state = state self.state_recreator = state_recreator def onHome(self): """Serve up the homepage.""" stateDict = self.state.__dict__.copy() stateDict.update(self.state.bayes.__dict__) statusTable = self.html.statusTable.clone() if not self.state.servers: statusTable.proxyDetails = "No POP3 proxies running." content = (self._buildBox('Status and Configuration', 'status.gif', statusTable % stateDict)+ self._buildTrainBox() + self._buildClassifyBox() + self._buildBox('Word query', 'query.gif', self.html.wordQuery) ) self._writePreamble("Home") self.write(content) self._writePostamble() def reReadOptions(self): """Called by the config page when the user saves some new options, or restores the defaults.""" # Reload the options. self.state.bayes.store() import Options reload(Options) global options from Options import options # Recreate the state. self.state = self.state_recreator() def verifyInput(self, parms): '''Check that the given input is valid.''' # Most of the work here is done by the parent class, but # we have a few extra checks errmsg = UserInterface.UserInterface.verifyInput(self, parms) # check for equal number of pop3servers and ports slist = list(parms['pop3proxy_remote_servers']) plist = list(parms['pop3proxy_listen_ports']) if len(slist) != len(plist): errmsg += '
  • The number of POP3 proxy ports specified ' + \ 'must match the number of servers specified
    • \n' # check for duplicate ports plist.sort() for p in range(len(plist)-1): try: if plist[p] == plist[p+1]: errmsg += '
  • All POP3 port numbers must be unique
    • ' break except IndexError: pass return errmsg Index: Options.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v retrieving revision 1.70 retrieving revision 1.71 diff -C2 -d -r1.70 -r1.71 *** Options.py 5 Sep 2003 01:15:28 -0000 1.70 --- Options.py 6 Sep 2003 04:03:35 -0000 1.71 *************** *** 892,895 **** --- 892,921 ---- ), + "imapserver" : ( + ("username", "Username", "", + """The username to use when logging into the SpamBayes IMAP server.""", + IMAP_ASTRING, DO_NOT_RESTORE), + + ("password", "Password", "", + """The password to use when logging into the SpamBayes IMAP server.""", + IMAP_ASTRING, DO_NOT_RESTORE), + + ("port", "IMAP Listen Port", 143, + """The port to serve the SpamBayes IMAP server on.""", + PORT, RESTORE), + + ("spam_directory", "Spam directory", "imapserver-spam", + """The directory to store spam messages in.""", + PATH, DO_NOT_RESTORE), + + ("ham_directory", "Ham directory", "imapserver-ham", + """The directory to store ham messages in.""", + PATH, DO_NOT_RESTORE), + + ("unsure_directory", "Unsure directory", "imapserver-unsure", + """The directory to store unsure messages in.""", + PATH, DO_NOT_RESTORE), + ), + "globals" : ( ("verbose", "Verbose", False, Index: Version.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/Version.py,v retrieving revision 1.17 retrieving revision 1.18 diff -C2 -d -r1.17 -r1.18 *** Version.py 2 Sep 2003 01:31:04 -0000 1.17 --- Version.py 6 Sep 2003 04:03:35 -0000 1.18 *************** *** 73,76 **** --- 73,85 ---- using %(InterfaceDescription)s, version %(InterfaceVersion)s""", }, + "IMAP Server" : { + "Version": 0.01, + "Description": "SpamBayes IMAP Server Alpha1", + "Date": "September 2003", + "InterfaceVersion": 0.02, + "InterfaceDescription": "SpamBayes IMAP Filter Web Interface Alpha2", + "Full Description": """%(Description)s, version %(Version)s (%(Date)s), + using %(InterfaceDescription)s, version %(InterfaceVersion)s""", + }, }, } From mhammond at users.sourceforge.net Sat Sep 6 21:53:16 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Sat Sep 6 23:53:19 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs async_processor.py, 1.7, 1.8 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs In directory sc8-pr-cvs1:/tmp/cvs-serv18595 Modified Files: async_processor.py Log Message: Handle ranges that don't quite have enough ticks or slightly too many. Don't use the underlying "tick" methods - just set the pos directly, as we know it. This fixes the problem that seemed to stop the progress bar from ever getting over 80% or so. Index: async_processor.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/async_processor.py,v retrieving revision 1.7 retrieving revision 1.8 diff -C2 -d -r1.7 -r1.8 *** async_processor.py 1 Sep 2003 05:34:19 -0000 1.7 --- async_processor.py 7 Sep 2003 03:53:13 -0000 1.8 *************** *** 13,16 **** --- 13,17 ---- True, False = 1, 0 + verbose = 0 IDC_START = 1100 *************** *** 50,54 **** win32api.PostMessage(self.hprogress, commctrl.PBM_SETSTEP, 1, 0) win32api.PostMessage(self.hprogress, commctrl.PBM_SETPOS, 0, 0) - self.current_control_tick = 0 self.current_stage += 1 --- 51,54 ---- *************** *** 59,68 **** def set_max_ticks(self, m): self._next_stage() - self.current_stage_tick = 0 self.current_stage_max = m def tick(self): ! self.current_stage_tick += 1 # Calc how far through this stage. this_prop = float(self.current_stage_tick) / self.current_stage_max --- 59,73 ---- def set_max_ticks(self, m): + # skip to the stage. self._next_stage() self.current_stage_max = m + self.current_stage_tick = -1 # ready to go to zero! + # if earlier stages stopped early, skip ahead. + self.tick() def tick(self): ! if self.current_stage_tick < self.current_stage_max: ! # Don't let us go beyond our stage max ! self.current_stage_tick += 1 # Calc how far through this stage. this_prop = float(self.current_stage_tick) / self.current_stage_max *************** *** 75,83 **** # user knows the process has actually started.) control_tick = max(1,int(total_prop * self.total_control_ticks)) ! #print "Tick", self.current_stage_tick, "is", this_prop, "through the stage,", total_prop, "through the total - ctrl tick is", control_tick ! while self.current_control_tick < control_tick: ! self.current_control_tick += 1 ! #print "ticking control", self.current_control_tick ! win32api.PostMessage(self.hprogress, commctrl.PBM_STEPIT, 0, 0) def _get_stage_text(self, text): --- 80,86 ---- # user knows the process has actually started.) control_tick = max(1,int(total_prop * self.total_control_ticks)) ! if verbose: ! print "Tick", self.current_stage_tick, "is", this_prop, "through the stage,", total_prop, "through the total - ctrl tick is", control_tick ! win32api.PostMessage(self.hprogress, commctrl.PBM_SETPOS, control_tick) def _get_stage_text(self, text): *************** *** 237,240 **** --- 240,244 ---- if __name__=='__main__': + verbose = 1 # Test my "multi-stage" code class HackProgress(_Progress): *************** *** 243,250 **** self.dlg = None self.stopping = False ! self.total_control_ticks = 100 self.current_stage = 0 self.set_stages( (("", 1.0),) ) p = HackProgress() p.set_max_ticks(10) --- 247,255 ---- self.dlg = None self.stopping = False ! self.total_control_ticks = 40 self.current_stage = 0 self.set_stages( (("", 1.0),) ) + print "Single stage test" p = HackProgress() p.set_max_ticks(10) *************** *** 252,255 **** --- 257,261 ---- p.tick() + print "First stage test" p = HackProgress() stages = ("Stage 1", 0.2), ("Stage 2", 0.8) *************** *** 260,265 **** p.tick() # Do stage 2 ! p.set_max_ticks(1000) ! for i in range(1000): p.tick() print "Done!" --- 266,292 ---- p.tick() # Do stage 2 ! p.set_max_ticks(20) ! for i in range(20): ! p.tick() ! print "Second stage test" ! p = HackProgress() ! stages = ("Stage 1", 0.9), ("Stage 2", 0.1) ! p.set_stages(stages) ! p.set_max_ticks(10) ! for i in range(7): # do a few less just to check ! p.tick() ! p.set_max_ticks(2) ! for i in range(2): ! p.tick() ! print "Third stage test" ! p = HackProgress() ! stages = ("Stage 1", 0.9), ("Stage 2", 0.1) ! p.set_stages(stages) ! p.set_max_ticks(300) ! for i in range(313): # do a few more just to check ! p.tick() ! p.set_max_ticks(2) ! for i in range(2): p.tick() + print "Done!" From mhammond at users.sourceforge.net Sun Sep 7 17:38:03 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Sun Sep 7 19:38:09 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000/docs troubleshooting.html, 1.14, 1.15 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000/docs In directory sc8-pr-cvs1:/tmp/cvs-serv14905/docs Modified Files: troubleshooting.html Log Message: Instructions how to locate the toolbar to delete assumed that the 'toolbars' tab would be the current tab. Index: troubleshooting.html =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/docs/troubleshooting.html,v retrieving revision 1.14 retrieving revision 1.15 diff -C2 -d -r1.14 -r1.15 *** troubleshooting.html 29 Aug 2003 06:24:48 -0000 1.14 --- troubleshooting.html 7 Sep 2003 23:38:01 -0000 1.15 *************** *** 46,50 ****
    • Right-click on any Outlook toolbar, and select Customize.
      • !
    • In the dialog that appears, locate SpamBayes in the list of toolbars, and select it.
    • Click on the Delete button.  Outlook will ask for --- 46,52 ----
      • Right-click on any Outlook toolbar, and select Customize.
        • !
      • In the dialog that appears, ensure the Toolbars tab is selected, locate ! SpamBayes in the list of toolbars, and select it.
      • Click on the Delete button.  Outlook will ask for From mhammond at users.sourceforge.net Sun Sep 7 17:42:22 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Sun Sep 7 19:42:25 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000 manager.py, 1.85, 1.86 addin.py, 1.107, 1.108 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000 In directory sc8-pr-cvs1:/tmp/cvs-serv15530 Modified Files: manager.py addin.py Log Message: When processing a hotmail inbox, we could see "object changed" exceptions reported by MAPI. Simply starting again on this exception seems to work. Index: manager.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/manager.py,v retrieving revision 1.85 retrieving revision 1.86 diff -C2 -d -r1.85 -r1.86 *** manager.py 5 Sep 2003 11:54:19 -0000 1.85 --- manager.py 7 Sep 2003 23:42:19 -0000 1.86 *************** *** 457,464 **** return print "ERROR:", repr(msg) - traceback.print_exc() if key in self.reported_error_map: print "(this error has already been reported - not displaying it again)" else: self.reported_error_map[key] = True ReportError(msg, title) --- 457,464 ---- return print "ERROR:", repr(msg) if key in self.reported_error_map: print "(this error has already been reported - not displaying it again)" else: + traceback.print_exc() self.reported_error_map[key] = True ReportError(msg, title) *************** *** 786,789 **** --- 786,791 ---- "must re-train the system via the SpamBayes manager." self.ReportErrorOnce(msg) + # and disable the addin, as we are hosed! + self.config.filter.enabled = False raise Index: addin.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/addin.py,v retrieving revision 1.107 retrieving revision 1.108 diff -C2 -d -r1.107 -r1.108 *** addin.py 4 Sep 2003 12:14:11 -0000 1.107 --- addin.py 7 Sep 2003 23:42:19 -0000 1.108 *************** *** 373,383 **** if not self.use_timer or not item.UnRead: ms = self.manager.message_store ! try: ! msgstore_message = ms.GetMessage(item) ! ProcessMessage(msgstore_message, self.manager) ! except ms.MsgStoreException, details: ! print "Unexpected error fetching message" ! traceback.print_exc() ! print details else: self._StartTimer() --- 373,397 ---- if not self.use_timer or not item.UnRead: ms = self.manager.message_store ! # The object can sometimes change underneath us (most noticably ! # with hotmail, which presumably has greater 'synchronization' ! # issues than tightly bound stores. We accept this error exactly ! # 2 times (once as we have seen it, and twice as it can't hurt ;) ! for i in range(3): ! try: ! msgstore_message = ms.GetMessage(item) ! ProcessMessage(msgstore_message, self.manager) ! break ! except ms.ObjectChangedException: ! # try again. ! self.manager.LogDebug(1, "Got object changed error - trying again...") ! continue ! except ms.MsgStoreException, details: ! print "Unexpected error fetching message" ! traceback.print_exc() ! print details ! break ! else: ! print "WARNING: Could not filter a message, as we got an " \ ! "'object changed' exception 3 times!" else: self._StartTimer() From mhammond at users.sourceforge.net Sun Sep 7 17:43:23 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Sun Sep 7 19:43:26 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000 msgstore.py,1.74,1.75 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000 In directory sc8-pr-cvs1:/tmp/cvs-serv15852 Modified Files: msgstore.py Log Message: Expose 'ObjectChanged' exception in the class as well as the module. Index: msgstore.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/msgstore.py,v retrieving revision 1.74 retrieving revision 1.75 diff -C2 -d -r1.74 -r1.75 *** msgstore.py 5 Sep 2003 11:54:19 -0000 1.74 --- msgstore.py 7 Sep 2003 23:43:21 -0000 1.75 *************** *** 154,157 **** --- 154,158 ---- NotFoundException = NotFoundException ReadOnlyException = ReadOnlyException + ObjectChangedException = ObjectChangedException def __init__(self, outlook = None): From mhammond at users.sourceforge.net Sun Sep 7 18:14:16 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Sun Sep 7 20:14:20 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000 addin.py,1.108,1.109 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000 In directory sc8-pr-cvs1:/tmp/cvs-serv20201 Modified Files: addin.py Log Message: Fix [ 798362 ] Toolbar becomes unresponsive and must be recreated If we fail to add an item to a popup, assume it is dead, so delete and re-create it. Index: addin.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/addin.py,v retrieving revision 1.108 retrieving revision 1.109 diff -C2 -d -r1.108 -r1.109 *** addin.py 7 Sep 2003 23:42:19 -0000 1.108 --- addin.py 8 Sep 2003 00:14:14 -0000 1.109 *************** *** 760,772 **** # The main tool-bar dropdown with all our entries. # Add a pop-up menu to the toolbar ! popup = self._AddControl( ! None, ! constants.msoControlPopup, ! None, None, ! Caption="SpamBayes", ! TooltipText = "SpamBayes anti-spam filters and functions", ! Enabled = True, ! Tag = "SpamBayesCommand.Popup") ! if popup is not None: # We may not be able to find/create our button # Convert from "CommandBarItem" to derived # "CommandBarPopup" Not sure if we should be able to work --- 760,778 ---- # The main tool-bar dropdown with all our entries. # Add a pop-up menu to the toolbar ! # but loop around twice - first time we may find a non-functioning button ! popup = None ! for attempt in range(2): ! popup = self._AddControl( ! None, ! constants.msoControlPopup, ! None, None, ! Caption="SpamBayes", ! TooltipText = "SpamBayes anti-spam filters and functions", ! Enabled = True, ! Tag = "SpamBayesCommand.Popup") ! if popup is None: ! # If the strategy below works for child buttons, we should ! # consider trying to re-create the top-level toolbar too. ! break # Convert from "CommandBarItem" to derived # "CommandBarPopup" Not sure if we should be able to work *************** *** 776,780 **** popup = CastTo(popup, "CommandBarPopup") # And add our children. ! self._AddControl(popup, constants.msoControlButton, ButtonEvent, (manager.ShowManager,), --- 782,786 ---- popup = CastTo(popup, "CommandBarPopup") # And add our children. ! child = self._AddControl(popup, constants.msoControlButton, ButtonEvent, (manager.ShowManager,), *************** *** 784,787 **** --- 790,812 ---- Visible=True, Tag = "SpamBayesCommand.Manager") + # Only necessary to check the first child - if the first works, + # the others will too + if child is None: + # Try and delete the popup, the bounce around the loop again, + # which will re-create it. + try: + item = self.CommandBars.FindControl( + Type = constants.msoControlPopup, + Tag = "SpamBayesCommand.Popup") + if item is None: + print "ERROR: Could't re-find control to delete" + break + item.Delete(False) + print "Deleted the dead popup control - re-creating" + except pythoncom.com_error, e: + print "ERROR: Failed to delete our dead toolbar control" + break + # ok - toolbar deleted - just run around the loop again + continue self._AddControl(popup, constants.msoControlButton, *************** *** 909,913 **** if toolbar.Name == "SpamBayes": self.toolbar = toolbar - print "Found SB toolbar - visible state is", toolbar.Visible break else: --- 934,937 ---- *************** *** 935,939 **** # eg, bug [ 755738 ] Latest CVS outllok doesn't work print "FAILED to add the toolbar item '%s' - %s" % (tag,e) - traceback.print_exc() return if image_fname: --- 959,962 ---- From mhammond at users.sourceforge.net Sun Sep 7 20:35:48 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Sun Sep 7 22:35:52 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000 tester.py,1.18,1.19 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000 In directory sc8-pr-cvs1:/tmp/cvs-serv9209 Modified Files: tester.py Log Message: Run every test over every "watch" folder - this is useful for testing hotmail/imap etc. IMAP still fails the test suite, but for reasons related to the test suite itself. Index: tester.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/tester.py,v retrieving revision 1.18 retrieving revision 1.19 diff -C2 -d -r1.18 -r1.19 *** tester.py 5 Sep 2003 07:21:14 -0000 1.18 --- tester.py 8 Sep 2003 02:35:46 -0000 1.19 *************** *** 140,158 **** folder = mgr.message_store.GetFolder(mgr.config.filter.unsure_folder_id) self.folder_unsure = folder.GetOutlookItem() ! # The "watch" folder is a folder we can stick stuff into to have them ! # filtered - just use the first one nominated. ! self.folder_watch = self.folder_watch_2 = None gen = mgr.message_store.GetFolderGenerator( mgr.config.filter.watch_folder_ids, mgr.config.filter.watch_include_sub) ! try: ! self.folder_watch = gen.next().GetOutlookItem() ! self.folder_watch_2 = gen.next().GetOutlookItem() ! except StopIteration: ! pass ! if self.folder_watch is None: ! raise RuntimeError, "Can't test without at least one folder to watch" ! # And the drafts folder where new messages are created. ! self.folder_drafts = mgr.outlook.Session.GetDefaultFolder(constants.olFolderDrafts) def FindTestMessage(self, folder): --- 140,153 ---- folder = mgr.message_store.GetFolder(mgr.config.filter.unsure_folder_id) self.folder_unsure = folder.GetOutlookItem() ! # And the drafts folder where new messages are created. ! self.folder_drafts = mgr.outlook.Session.GetDefaultFolder(constants.olFolderDrafts) ! ! def GetWatchFolderGenerator(self): ! mgr = self.manager gen = mgr.message_store.GetFolderGenerator( mgr.config.filter.watch_folder_ids, mgr.config.filter.watch_include_sub) ! for f in gen: ! yield f, f.GetOutlookItem() def FindTestMessage(self, folder): *************** *** 161,182 **** return items.Find("[Subject] = '%s'" % (subject,)) def _CleanTestMessageFromFolder(self, folder): subject = TEST_SUBJECT num = 0 ! while True: msg = self.FindTestMessage(folder) if msg is None: break msg.Delete() ! num += 1 if num: print "Cleaned %d test messages from folder '%s'" % (num, folder.Name) def CleanAllTestMessages(self): - subject = TEST_SUBJECT self._CleanTestMessageFromFolder(self.folder_spam) self._CleanTestMessageFromFolder(self.folder_unsure) - self._CleanTestMessageFromFolder(self.folder_watch) self._CleanTestMessageFromFolder(self.folder_drafts) def CreateTestMessageInFolder(self, spam_status, folder): --- 156,192 ---- return items.Find("[Subject] = '%s'" % (subject,)) + def CheckMessageFilteredFrom(self, folder): + # For hotmail accounts, the message may take a little time to actually + # be removed from the original folder (ie, it appears in the "dest" + # folder before it vanished. + for i in range(5): + if self.FindTestMessage(folder) is None: + break + sleep(.5) + else: + TestFailed("The test message remained in folder %r" % folder.Name) + def _CleanTestMessageFromFolder(self, folder): subject = TEST_SUBJECT num = 0 ! # imap/hotmail etc only soft delete, and I see no way to differentiate ! # force the user to purge them manually ! for i in range(50): msg = self.FindTestMessage(folder) if msg is None: break msg.Delete() ! else: ! raise TestFailed("Old test messages appear to still exist. These may" \ ! "be 'soft-deleted' - you will need to purge them manually") if num: print "Cleaned %d test messages from folder '%s'" % (num, folder.Name) def CleanAllTestMessages(self): self._CleanTestMessageFromFolder(self.folder_spam) self._CleanTestMessageFromFolder(self.folder_unsure) self._CleanTestMessageFromFolder(self.folder_drafts) + for msf, of in self.GetWatchFolderGenerator(): + self._CleanTestMessageFromFolder(of) def CreateTestMessageInFolder(self, spam_status, folder): *************** *** 223,327 **** nham = bayes.nham original_bayes = copy.copy(driver.manager.classifier_data.bayes) ! # Create a spam message in the Inbox - it should get immediately filtered ! msg, words = driver.CreateTestMessageInFolder(SPAM, driver.folder_watch) ! # sleep to ensure filtering. ! WaitForFilters() ! # It should no longer be in the Inbox. ! if driver.FindTestMessage(driver.folder_watch) is not None: ! TestFailed("The test message appeared to not be filtered") ! # It should be in the "sure is spam" folder. ! spam_msg = driver.FindTestMessage(driver.folder_spam) ! if spam_msg is None: ! TestFailed("The test message vanished from the Inbox, but didn't appear in Spam") ! # Check that none of the above caused training. ! if nspam != bayes.nspam: ! TestFailed("Something caused a new spam message to appear") ! if nham != bayes.nham: ! TestFailed("Something caused a new ham message to appear") ! check_words(words, bayes, 0, 0) ! ! # Now move the message back to the inbox - it should get trained. ! store_msg = driver.manager.message_store.GetMessage(spam_msg) ! import train ! if train.been_trained_as_ham(store_msg, driver.manager.classifier_data): ! TestFailed("This new spam message should not have been trained as ham yet") ! if train.been_trained_as_spam(store_msg, driver.manager.classifier_data): ! TestFailed("This new spam message should not have been trained as spam yet") ! spam_msg.Move(driver.folder_watch) ! WaitForFilters() ! spam_msg = driver.FindTestMessage(driver.folder_watch) ! if spam_msg is None: ! TestFailed("The message appears to have been filtered out of the watch folder") ! store_msg = driver.manager.message_store.GetMessage(spam_msg) ! need_untrain = True ! try: ! if nspam != bayes.nspam: ! TestFailed("There were not the same number of spam messages after a re-train") ! if nham+1 != bayes.nham: ! TestFailed("There was not one more ham messages after a re-train") ! if train.been_trained_as_spam(store_msg, driver.manager.classifier_data): ! TestFailed("This new spam message should not have been trained as spam yet") ! if not train.been_trained_as_ham(store_msg, driver.manager.classifier_data): ! TestFailed("This new spam message should have been trained as ham now") ! # word infos should have one extra ham ! check_words(words, bayes, 0, 1) ! # Now move it back to the Spam folder. ! # This should see the message un-trained as ham, and re-trained as Spam ! spam_msg.Move(driver.folder_spam) WaitForFilters() spam_msg = driver.FindTestMessage(driver.folder_spam) if spam_msg is None: ! TestFailed("Could not find the message in the Spam folder") ! store_msg = driver.manager.message_store.GetMessage(spam_msg) ! if nspam +1 != bayes.nspam: ! TestFailed("There should be one more spam now") if nham != bayes.nham: ! TestFailed("There should be the same number of hams again") ! if not train.been_trained_as_spam(store_msg, driver.manager.classifier_data): ! TestFailed("This new spam message should have been trained as spam by now") if train.been_trained_as_ham(store_msg, driver.manager.classifier_data): ! TestFailed("This new spam message should have been un-trained as ham") ! # word infos should have one extra spam, no extra ham ! check_words(words, bayes, 1, 0) ! # Move the message to another folder, and make sure we still ! # identify it correctly as having been trained. ! # Move to the "unsure" folder, just cos we know about it, and ! # we know that no special watching of this folder exists. ! spam_msg.Move(driver.folder_unsure) ! spam_msg = driver.FindTestMessage(driver.folder_unsure) if spam_msg is None: ! TestFailed("Could not find the message in the Unsure folder") store_msg = driver.manager.message_store.GetMessage(spam_msg) ! if not train.been_trained_as_spam(store_msg, driver.manager.classifier_data): ! TestFailed("Message was not identified as Spam after moving") ! ! # word infos still be 'spam' ! check_words(words, bayes, 1, 0) ! ! # Now undo the damage we did. ! was_spam = train.untrain_message(store_msg, driver.manager.classifier_data) ! if not was_spam: ! TestFailed("Untraining this message did not indicate it was spam") ! if train.been_trained_as_spam(store_msg, driver.manager.classifier_data) or \ ! train.been_trained_as_ham(store_msg, driver.manager.classifier_data): ! TestFailed("Untraining this message kept it has ham/spam") ! need_untrain = False ! finally: ! if need_untrain: ! train.untrain_message(store_msg, driver.manager.classifier_data) ! ! # Check all the counts are back where we started. ! if nspam != bayes.nspam: ! TestFailed("Spam count didn't get back to the same") ! if nham != bayes.nham: ! TestFailed("Ham count didn't get back to the same") ! check_words(words, bayes, 0, 0) ! ! if bayes.wordinfo != original_bayes.wordinfo: ! TestFailed("The bayes object's 'wordinfo' did not compare the same at the end of all this!") ! if bayes.probcache != original_bayes.probcache: ! TestFailed("The bayes object's 'probcache' did not compare the same at the end of all this!") ! ! spam_msg.Delete() print "Created a Spam message, and saw it get filtered and trained." --- 233,339 ---- nham = bayes.nham original_bayes = copy.copy(driver.manager.classifier_data.bayes) ! # for each watch folder, create a spam message, and do the training thang ! for msf_watch, folder_watch in driver.GetWatchFolderGenerator(): ! print "Performing Spam test on watch folder '%s'..." % msf_watch.GetFQName() ! # Create a spam message in the Inbox - it should get immediately filtered ! msg, words = driver.CreateTestMessageInFolder(SPAM, folder_watch) ! # sleep to ensure filtering. WaitForFilters() + # It should no longer be in the Inbox. + driver.CheckMessageFilteredFrom(folder_watch) + # It should be in the "sure is spam" folder. spam_msg = driver.FindTestMessage(driver.folder_spam) if spam_msg is None: ! TestFailed("The test message vanished from the Inbox, but didn't appear in Spam") ! # Check that none of the above caused training. ! if nspam != bayes.nspam: ! TestFailed("Something caused a new spam message to appear") if nham != bayes.nham: ! TestFailed("Something caused a new ham message to appear") ! check_words(words, bayes, 0, 0) ! ! # Now move the message back to the inbox - it should get trained. ! store_msg = driver.manager.message_store.GetMessage(spam_msg) ! import train if train.been_trained_as_ham(store_msg, driver.manager.classifier_data): ! TestFailed("This new spam message should not have been trained as ham yet") ! if train.been_trained_as_spam(store_msg, driver.manager.classifier_data): ! TestFailed("This new spam message should not have been trained as spam yet") ! spam_msg.Move(folder_watch) ! WaitForFilters() ! spam_msg = driver.FindTestMessage(folder_watch) if spam_msg is None: ! TestFailed("The message appears to have been filtered out of the watch folder") store_msg = driver.manager.message_store.GetMessage(spam_msg) ! need_untrain = True ! try: ! if nspam != bayes.nspam: ! TestFailed("There were not the same number of spam messages after a re-train") ! if nham+1 != bayes.nham: ! TestFailed("There was not one more ham messages after a re-train") ! if train.been_trained_as_spam(store_msg, driver.manager.classifier_data): ! TestFailed("This new spam message should not have been trained as spam yet") ! if not train.been_trained_as_ham(store_msg, driver.manager.classifier_data): ! TestFailed("This new spam message should have been trained as ham now") ! # word infos should have one extra ham ! check_words(words, bayes, 0, 1) ! # Now move it back to the Spam folder. ! # This should see the message un-trained as ham, and re-trained as Spam ! spam_msg.Move(driver.folder_spam) ! WaitForFilters() ! spam_msg = driver.FindTestMessage(driver.folder_spam) ! if spam_msg is None: ! TestFailed("Could not find the message in the Spam folder") ! store_msg = driver.manager.message_store.GetMessage(spam_msg) ! if nspam +1 != bayes.nspam: ! TestFailed("There should be one more spam now") ! if nham != bayes.nham: ! TestFailed("There should be the same number of hams again") ! if not train.been_trained_as_spam(store_msg, driver.manager.classifier_data): ! TestFailed("This new spam message should have been trained as spam by now") ! if train.been_trained_as_ham(store_msg, driver.manager.classifier_data): ! TestFailed("This new spam message should have been un-trained as ham") ! # word infos should have one extra spam, no extra ham ! check_words(words, bayes, 1, 0) ! # Move the message to another folder, and make sure we still ! # identify it correctly as having been trained. ! # Move to the "unsure" folder, just cos we know about it, and ! # we know that no special watching of this folder exists. ! spam_msg.Move(driver.folder_unsure) ! spam_msg = driver.FindTestMessage(driver.folder_unsure) ! if spam_msg is None: ! TestFailed("Could not find the message in the Unsure folder") ! store_msg = driver.manager.message_store.GetMessage(spam_msg) ! if not train.been_trained_as_spam(store_msg, driver.manager.classifier_data): ! TestFailed("Message was not identified as Spam after moving") ! ! # word infos still be 'spam' ! check_words(words, bayes, 1, 0) ! ! # Now undo the damage we did. ! was_spam = train.untrain_message(store_msg, driver.manager.classifier_data) ! if not was_spam: ! TestFailed("Untraining this message did not indicate it was spam") ! if train.been_trained_as_spam(store_msg, driver.manager.classifier_data) or \ ! train.been_trained_as_ham(store_msg, driver.manager.classifier_data): ! TestFailed("Untraining this message kept it has ham/spam") ! need_untrain = False ! finally: ! if need_untrain: ! train.untrain_message(store_msg, driver.manager.classifier_data) ! ! # Check all the counts are back where we started. ! if nspam != bayes.nspam: ! TestFailed("Spam count didn't get back to the same") ! if nham != bayes.nham: ! TestFailed("Ham count didn't get back to the same") ! check_words(words, bayes, 0, 0) ! ! if bayes.wordinfo != original_bayes.wordinfo: ! TestFailed("The bayes object's 'wordinfo' did not compare the same at the end of all this!") ! if bayes.probcache != original_bayes.probcache: ! TestFailed("The bayes object's 'probcache' did not compare the same at the end of all this!") ! ! spam_msg.Delete() print "Created a Spam message, and saw it get filtered and trained." *************** *** 340,345 **** WaitForFilters() # It should still be in the Inbox. ! if driver.FindTestMessage(folder1) is None: ! TestFailed("The test ham message appeared to have been filtered!") # Manually move it to folder2 --- 352,356 ---- WaitForFilters() # It should still be in the Inbox. ! driver.CheckMessageFilteredFrom(folder1) # Manually move it to folder2 *************** *** 381,395 **** def TestUnsureFilter(driver): # Create a spam message in the Inbox - it should get immediately filtered ! msg, words = driver.CreateTestMessageInFolder(UNSURE, driver.folder_watch) ! # sleep to ensure filtering. ! WaitForFilters() ! # It should no longer be in the Inbox. ! if driver.FindTestMessage(driver.folder_watch) is not None: ! TestFailed("The test unsure message appeared to not be filtered") ! # It should be in the "unsure" folder. ! spam_msg = driver.FindTestMessage(driver.folder_unsure) ! if spam_msg is None: ! TestFailed("The test message vanished from the Inbox, but didn't appear in Unsure") ! spam_msg.Delete() print "Created an unsure message, and saw it get filtered" --- 392,407 ---- def TestUnsureFilter(driver): # Create a spam message in the Inbox - it should get immediately filtered ! for msf_watch, folder_watch in driver.GetWatchFolderGenerator(): ! print "Performing Spam test on watch folder '%s'..." % msf_watch.GetFQName() ! msg, words = driver.CreateTestMessageInFolder(UNSURE, folder_watch) ! # sleep to ensure filtering. ! WaitForFilters() ! # It should no longer be in the Inbox. ! driver.CheckMessageFilteredFrom(folder_watch) ! # It should be in the "unsure" folder. ! spam_msg = driver.FindTestMessage(driver.folder_unsure) ! if spam_msg is None: ! TestFailed("The test message vanished from the Inbox, but didn't appear in Unsure") ! spam_msg.Delete() print "Created an unsure message, and saw it get filtered" *************** *** 549,571 **** checkpoint) # message moved after we have ID, but before opening. ! folder = driver.folder_watch ! if is_ham: ! msg, words = driver.CreateTestMessageInFolder(HAM, folder) ! else: ! msg, words = driver.CreateTestMessageInFolder(SPAM, folder) ! try: ! _setup_for_mapi_failure(checkpoint, hr) try: ! # sleep to ensure filtering. ! WaitForFilters() finally: ! _restore_mapi_failure() ! if driver.FindTestMessage(folder) is None: ! TestFailed("We appear to have filtered a message even though we forced 'not found' failure") ! finally: ! print "<- Finished MAPI error '%s' in %s" % (mapiutil.GetScodeString(hr), ! checkpoint) ! if msg is not None: ! msg.Delete() def do_failure_tests(manager): --- 561,584 ---- checkpoint) # message moved after we have ID, but before opening. ! for msf, folder in driver.GetWatchFolderGenerator(): ! print "Testing in folder '%s'" % msf.GetFQName() ! if is_ham: ! msg, words = driver.CreateTestMessageInFolder(HAM, folder) ! else: ! msg, words = driver.CreateTestMessageInFolder(SPAM, folder) try: ! _setup_for_mapi_failure(checkpoint, hr) ! try: ! # sleep to ensure filtering. ! WaitForFilters() ! finally: ! _restore_mapi_failure() ! if driver.FindTestMessage(folder) is None: ! TestFailed("We appear to have filtered a message even though we forced 'not found' failure") finally: ! if msg is not None: ! msg.Delete() ! print "<- Finished MAPI error '%s' in %s" % (mapiutil.GetScodeString(hr), ! checkpoint) def do_failure_tests(manager): *************** *** 620,628 **** assert not manager.test_suite_running, "already running??" manager.test_suite_running = True - run_nonfilter_tests(manager) - # filtering tests take alot of time - do them last. run_filter_tests(manager) run_failure_tests(manager) run_invalid_id_tests(manager) print "*" * 20 print "Test suite finished without error!" --- 633,641 ---- assert not manager.test_suite_running, "already running??" manager.test_suite_running = True run_filter_tests(manager) run_failure_tests(manager) run_invalid_id_tests(manager) + # non-filter tests take alot of time - do them last. + run_nonfilter_tests(manager) print "*" * 20 print "Test suite finished without error!" From anadelonbrin at users.sourceforge.net Mon Sep 8 01:04:19 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon Sep 8 03:04:24 2003 Subject: [Spambayes-checkins] spambayes/scripts sb_imapfilter.py,1.2,1.3 Message-ID: Update of /cvsroot/spambayes/spambayes/scripts In directory sc8-pr-cvs1:/tmp/cvs-serv14300/scripts Modified Files: sb_imapfilter.py Log Message: Fix two bugs found by Rafael Scholl. Index: sb_imapfilter.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/scripts/sb_imapfilter.py,v retrieving revision 1.2 retrieving revision 1.3 diff -C2 -d -r1.2 -r1.3 *** sb_imapfilter.py 5 Sep 2003 01:25:27 -0000 1.2 --- sb_imapfilter.py 8 Sep 2003 07:04:16 -0000 1.3 *************** *** 486,492 **** response = imap.uid("SEARCH", "UNDELETED") self._check(response, "SEARCH UNDELETED") ! if response[1][0] == "": return [] - return response[1][0].split(' ') def __getitem__(self, key): --- 486,493 ---- response = imap.uid("SEARCH", "UNDELETED") self._check(response, "SEARCH UNDELETED") ! if response[1][0]: ! return response[1][0].split(' ') ! else: return [] def __getitem__(self, key): From anadelonbrin at users.sourceforge.net Mon Sep 8 01:04:20 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Mon Sep 8 03:04:28 2003 Subject: [Spambayes-checkins] spambayes/spambayes UserInterface.py, 1.23, 1.24 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv14300/spambayes Modified Files: UserInterface.py Log Message: Fix two bugs found by Rafael Scholl. Index: UserInterface.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/UserInterface.py,v retrieving revision 1.23 retrieving revision 1.24 diff -C2 -d -r1.23 -r1.24 *** UserInterface.py 1 Sep 2003 10:33:48 -0000 1.23 --- UserInterface.py 8 Sep 2003 07:04:17 -0000 1.24 *************** *** 654,662 **** def onChangeopts(self, **parms): if parms.has_key("how"): if parms["how"] == "Save advanced options": pmap = self.advanced_options_map - else: - pmap = self.parm_ini_map del parms["how"] html = self._getHTMLClone() --- 654,661 ---- def onChangeopts(self, **parms): + pmap = self.parm_ini_map if parms.has_key("how"): if parms["how"] == "Save advanced options": pmap = self.advanced_options_map del parms["how"] html = self._getHTMLClone() From mhammond at users.sourceforge.net Mon Sep 8 01:51:54 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Mon Sep 8 03:51:58 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000 tester.py,1.19,1.20 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000 In directory sc8-pr-cvs1:/tmp/cvs-serv22578 Modified Files: tester.py Log Message: Accidently lost a call to a specific incremental training test - reinstate it (and a few more tweaks trying to get it all running on an IMAP inbox) Index: tester.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/tester.py,v retrieving revision 1.19 retrieving revision 1.20 diff -C2 -d -r1.19 -r1.20 *** tester.py 8 Sep 2003 02:35:46 -0000 1.19 --- tester.py 8 Sep 2003 07:51:51 -0000 1.20 *************** *** 50,54 **** def WaitForFilters(): - import pythoncom # Must wait longer than normal, so when run with a timer we still work. filter_event.clear() --- 50,53 ---- *************** *** 163,169 **** if self.FindTestMessage(folder) is None: break ! sleep(.5) else: ! TestFailed("The test message remained in folder %r" % folder.Name) def _CleanTestMessageFromFolder(self, folder): --- 162,170 ---- if self.FindTestMessage(folder) is None: break ! for j in range(10): ! sleep(.05) else: ! ms_folder = self.manager.message_store.GetFolder(folder) ! TestFailed("The test message remained in folder '%s'" % ms_folder.GetFQName()) def _CleanTestMessageFromFolder(self, folder): *************** *** 352,364 **** WaitForFilters() # It should still be in the Inbox. ! driver.CheckMessageFilteredFrom(folder1) # Manually move it to folder2 msg.Move(folder2) - # re-find it in folder2 - msg = driver.FindTestMessage(folder2) - # sleep to any processing in this folder. WaitForFilters() if nspam != bayes.nspam or nham != bayes.nham: --- 353,367 ---- WaitForFilters() # It should still be in the Inbox. ! if driver.FindTestMessage(folder1) is None: ! TestFailed("The test ham message appeared to have been filtered!") # Manually move it to folder2 msg.Move(folder2) # sleep to any processing in this folder. WaitForFilters() + # re-find it in folder2 + msg = driver.FindTestMessage(folder2) + if driver.FindTestMessage(folder2) is None: + TestFailed("Couldn't find the ham message we just moved") if nspam != bayes.nspam or nham != bayes.nham: *************** *** 383,386 **** --- 386,390 ---- mgr.config.filter.watch_include_sub) num = 0 + folders = [] for f in gen: print "Running ham filter tests on folder '%s'" % f.GetFQName() *************** *** 388,391 **** --- 392,406 ---- _DoTestHamFilter(driver, f) num += 1 + folders.append(f) + # Now test incremental train logic, between all these folders. + if len(folders)<2: + print "NOTE: Can't do incremental training tests as only 1 watch folder is in place" + else: + for f in folders: + # 'targets' is a list of all folders except this + targets = folders[:] + targets.remove(f) + for t in targets: + _DoTestHamTrain(driver, f, t) print "Created a Ham message, and saw it remain in place (in %d watch folders.)" % num *************** *** 614,621 **** def filter_message_with_event(msg, mgr, all_actions=True): import filter ! try: ! return filter._original_filter_message(msg, mgr, all_actions) ! finally: ! filter_event.set() def test(manager): --- 629,636 ---- def filter_message_with_event(msg, mgr, all_actions=True): import filter ! ret = filter._original_filter_message(msg, mgr, all_actions) ! if ret != "Failed": ! filter_event.set() # only set if it works ! return ret def test(manager): From mhammond at users.sourceforge.net Mon Sep 8 01:53:08 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Mon Sep 8 03:53:12 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs/resources dialogs.rc, 1.37, 1.38 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs/resources In directory sc8-pr-cvs1:/tmp/cvs-serv22804 Modified Files: dialogs.rc Log Message: Correct grammar on dialog. Index: dialogs.rc =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/resources/dialogs.rc,v retrieving revision 1.37 retrieving revision 1.38 diff -C2 -d -r1.37 -r1.38 *** dialogs.rc 5 Sep 2003 06:51:00 -0000 1.37 --- dialogs.rc 8 Sep 2003 07:53:05 -0000 1.38 *************** *** 42,46 **** CONTROL "",IDC_DELAY2_SLIDER,"msctls_trackbar32",TBS_AUTOTICKS | TBS_TOP | WS_TABSTOP,16,73,148,22 ! LTEXT "Delay between new mail check",IDC_STATIC,16,62,101,8 EDITTEXT IDC_DELAY2_TEXT,165,79,40,14,ES_AUTOHSCROLL LTEXT "seconds",IDC_STATIC,207,82,28,8 --- 42,46 ---- CONTROL "",IDC_DELAY2_SLIDER,"msctls_trackbar32",TBS_AUTOTICKS | TBS_TOP | WS_TABSTOP,16,73,148,22 ! LTEXT "Delay between processing items",IDC_STATIC,16,62,142,8 EDITTEXT IDC_DELAY2_TEXT,165,79,40,14,ES_AUTOHSCROLL LTEXT "seconds",IDC_STATIC,207,82,28,8 From mhammond at users.sourceforge.net Mon Sep 8 01:53:49 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Mon Sep 8 03:53:52 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000/installer spambayes_addin.iss, 1.9, 1.10 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000/installer In directory sc8-pr-cvs1:/tmp/cvs-serv23184 Modified Files: spambayes_addin.iss Log Message: Installed "welcome.html" into the wrong place Index: spambayes_addin.iss =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/installer/spambayes_addin.iss,v retrieving revision 1.9 retrieving revision 1.10 diff -C2 -d -r1.9 -r1.10 *** spambayes_addin.iss 4 Sep 2003 11:59:35 -0000 1.9 --- spambayes_addin.iss 8 Sep 2003 07:53:46 -0000 1.10 *************** *** 18,22 **** Source: "dist\spambayes_addin.dll"; DestDir: "{app}"; Flags: ignoreversion regserver Source: "dist\*.*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs ! Source: "dist\docs\welcome.html"; DestDir: "{app}"; Flags: isreadme [UninstallDelete] --- 18,22 ---- Source: "dist\spambayes_addin.dll"; DestDir: "{app}"; Flags: ignoreversion regserver Source: "dist\*.*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs ! Source: "dist\docs\welcome.html"; DestDir: "{app}\docs"; Flags: isreadme [UninstallDelete] From mhammond at users.sourceforge.net Mon Sep 8 01:56:52 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Mon Sep 8 03:56:55 2003 Subject: [Spambayes-checkins] spambayes/spambayes Version.py,1.18,1.19 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv23584 Modified Files: Version.py Log Message: Outlook moves to 0.8, and the download location changes to http://spambayes.sourceforge.net/windows.html Index: Version.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/Version.py,v retrieving revision 1.18 retrieving revision 1.19 diff -C2 -d -r1.18 -r1.19 *** Version.py 6 Sep 2003 04:03:35 -0000 1.18 --- Version.py 8 Sep 2003 07:56:50 -0000 1.19 *************** *** 31,38 **** }, "Outlook" : { ! "Version": 0.7, ! "BinaryVersion": 0.7, "Description": "SpamBayes Outlook Addin", ! "Date": "August 9, 2003", "Full Description": "%(Description)s, version %(Version)s (%(Date)s)", "Full Description Binary": --- 31,38 ---- }, "Outlook" : { ! "Version": 0.8, ! "BinaryVersion": 0.8, "Description": "SpamBayes Outlook Addin", ! "Date": "September 8, 2003", "Full Description": "%(Description)s, version %(Version)s (%(Date)s)", "Full Description Binary": *************** *** 41,45 **** # versions will still go to the new page. # We may also like to have a "Release Notes Page" item later? ! "Download Page": "http://starship.python.net/crew/mhammond/spambayes" }, "POP3 Proxy" : { --- 41,45 ---- # versions will still go to the new page. # We may also like to have a "Release Notes Page" item later? ! "Download Page": "http://spambayes.sourceforge.net/windows.html" }, "POP3 Proxy" : { From mhammond at users.sourceforge.net Mon Sep 8 02:17:31 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Mon Sep 8 04:17:37 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000/installer spambayes_addin.iss, 1.10, 1.11 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000/installer In directory sc8-pr-cvs1:/tmp/cvs-serv27220 Modified Files: spambayes_addin.iss Log Message: Version 008 Index: spambayes_addin.iss =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/installer/spambayes_addin.iss,v retrieving revision 1.10 retrieving revision 1.11 diff -C2 -d -r1.10 -r1.11 *** spambayes_addin.iss 8 Sep 2003 07:53:46 -0000 1.10 --- spambayes_addin.iss 8 Sep 2003 08:17:29 -0000 1.11 *************** *** 5,10 **** [Setup] AppName=Spambayes Outlook Addin ! AppVerName=Spambayes Outlook Addin 0.7 ! AppVersion=0.7 DefaultDirName={pf}\Spambayes Outlook Addin DefaultGroupName=Spambayes Outlook Addin --- 5,10 ---- [Setup] AppName=Spambayes Outlook Addin ! AppVerName=Spambayes Outlook Addin 0.8 ! AppVersion=0.8 DefaultDirName={pf}\Spambayes Outlook Addin DefaultGroupName=Spambayes Outlook Addin From sjoerd at users.sourceforge.net Mon Sep 8 03:19:46 2003 From: sjoerd at users.sourceforge.net (Sjoerd Mullender) Date: Mon Sep 8 05:19:52 2003 Subject: [Spambayes-checkins] spambayes/scripts sb_imapfilter.py,1.3,1.4 Message-ID: Update of /cvsroot/spambayes/spambayes/scripts In directory sc8-pr-cvs1:/tmp/cvs-serv5793 Modified Files: sb_imapfilter.py Log Message: Trivial fix for IMAP over SSL. Index: sb_imapfilter.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/scripts/sb_imapfilter.py,v retrieving revision 1.3 retrieving revision 1.4 diff -C2 -d -r1.3 -r1.4 *** sb_imapfilter.py 8 Sep 2003 07:04:16 -0000 1.3 --- sb_imapfilter.py 8 Sep 2003 09:19:44 -0000 1.4 *************** *** 111,115 **** try: if options["imap", "use_ssl"]: ! from imaplib import IMAP_SSL as BaseIMAP else: from imaplib import IMAP4 as BaseIMAP --- 111,115 ---- try: if options["imap", "use_ssl"]: ! from imaplib import IMAP4_SSL as BaseIMAP else: from imaplib import IMAP4 as BaseIMAP From ta-meyer at ihug.co.nz Mon Sep 8 22:22:47 2003 From: ta-meyer at ihug.co.nz (Tony Meyer) Date: Mon Sep 8 05:22:52 2003 Subject: [Spambayes-checkins] spambayes/scripts sb_imapfilter.py,1.3,1.4 In-Reply-To: <1ED4ECF91CDED24C8D012BCF2B034F13031B7CA6@its-xchg4.massey.ac.nz> Message-ID: <1ED4ECF91CDED24C8D012BCF2B034F130212AE53@its-xchg4.massey.ac.nz> > Trivial fix for IMAP over SSL. [...] > ! from imaplib import IMAP_SSL as BaseIMAP [...] > ! from imaplib import IMAP4_SSL as BaseIMAP . Thanks. (it does say, "untested" ) =Tony Meyer From mhammond at users.sourceforge.net Mon Sep 8 04:56:00 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Mon Sep 8 06:56:09 2003 Subject: [Spambayes-checkins] website applications.ht, 1.21, 1.22 download.ht, 1.17, 1.18 index.ht, 1.24, 1.25 windows.ht, 1.26, 1.27 Message-ID: Update of /cvsroot/spambayes/website In directory sc8-pr-cvs1:/tmp/cvs-serv22912 Modified Files: applications.ht download.ht index.ht windows.ht Log Message: Outlook Version 008. Index: applications.ht =================================================================== RCS file: /cvsroot/spambayes/website/applications.ht,v retrieving revision 1.21 retrieving revision 1.22 diff -C2 -d -r1.21 -r1.22 *** applications.ht 5 Sep 2003 08:02:54 -0000 1.21 --- applications.ht 8 Sep 2003 10:55:58 -0000 1.22 *************** *** 28,33 ****

        Availability

        Mark has packaged together an installer for the plugin. ! Go to the Windows page for more. ! This is currently at version 007.

        Download the alpha4 release.

        Alternatively, you can use CVS to get the code - go to the CVS page on the project's sourceforge site for more.

        --- 28,32 ----

        Availability

        Mark has packaged together an installer for the plugin. ! Go to the Windows page for more.

        Download the alpha4 release.

        Alternatively, you can use CVS to get the code - go to the CVS page on the project's sourceforge site for more.

        Index: download.ht =================================================================== RCS file: /cvsroot/spambayes/website/download.ht,v retrieving revision 1.17 retrieving revision 1.18 diff -C2 -d -r1.17 -r1.18 *** download.ht 5 Sep 2003 08:02:54 -0000 1.17 --- download.ht 8 Sep 2003 10:55:58 -0000 1.18 *************** *** 24,29 ****

        Outlook Plugin

        Mark has packaged up a standalone installer of the Outlook plugin. Get it ! from the Windows page on this site. ! This is currently at version 007.

        Other

        --- 24,28 ----

        Outlook Plugin

        Mark has packaged up a standalone installer of the Outlook plugin. Get it ! from the Windows page on this site.

        Other

        Index: index.ht =================================================================== RCS file: /cvsroot/spambayes/website/index.ht,v retrieving revision 1.24 retrieving revision 1.25 diff -C2 -d -r1.24 -r1.25 *** index.ht 4 Sep 2003 21:35:29 -0000 1.24 --- index.ht 8 Sep 2003 10:55:58 -0000 1.25 *************** *** 5,9 ****

        News

        !

        Outlook plugin release 007 is available. See the Windows page for more. Note that the Outlook plugin is now available from Sourceforge, on this project's Files page.

        Fifth pre-release of spambayes available. See the download page for more.

        You may also like to see what other people have been saying about us in the press and elsewhere.

        --- 5,9 ----

        News

        !

        Outlook plugin release 008 is available. See the Windows page for more. Note that the Outlook plugin is now available from Sourceforge, on this project's Files page.

        Fifth pre-release of spambayes available. See the download page for more.

        You may also like to see what other people have been saying about us in the press and elsewhere.

        Index: windows.ht =================================================================== RCS file: /cvsroot/spambayes/website/windows.ht,v retrieving revision 1.26 retrieving revision 1.27 diff -C2 -d -r1.26 -r1.27 *** windows.ht 13 Aug 2003 13:42:10 -0000 1.26 --- windows.ht 8 Sep 2003 10:55:58 -0000 1.27 *************** *** 5,23 ****

        Spambayes on Windows

        !

        Outlook 2000/Outlook XP

        !

        If you are using Outlook 2000 or Outlook XP (not Outlook ! Express) you should be able to simply download and run the ! Outlook ! plug-in installer (alternate ! download site). A separate Python installation is not necessary. ! Mark's ! website has more details on this package.

        !

        Note that all users who installed version 006 or earlier of the plugin are ! recommended to upgrade to the 007 release.

        !

        The Outlook add-in works with all versions of Windows, and all ! versions of Outlook 2000 and later - see below for more details.

        If you have a problem, the best way to help make progress is to look for --- 5,29 ----

        Spambayes on Windows

        !

        This page has information for users of Microsoft Outlook, ! and for all other mail clients !

        Outlook 2000/Outlook XP

        !

        Latest Release

        !

        The latest release is Version 008 - see the ! release notes ! or download the installation program. !

        !

        General Information

        !

        ! The Outlook addin is an application of the SpamBayes project. It works with ! Outlook 2000 and later on Windows 98 and later. It does not work with ! Outlook Express - see below for other options !

        !

        In general, you should download and install the latest version, as shown ! above. Older versions and the release notes can be viewed at the ! spambayes releases page. !

        If you have a problem, the best way to help make progress is to look for *************** *** 28,40 **** the ! troubleshooting.html file that is installed with the plugin.

        ! !

        Compatibility

        ! !

        As far as we know, the latest release of the plugin works with all ! versions of Windows and Outlook 2000 and later (Outlook 2003 should have ! the latest Technical Release installed). If you have a problem, please follow ! the bug reporting steps above; we will endeavour to keep this page ! up to date.

        Installing the Outlook Client From Source

        --- 34,38 ---- the ! troubleshooting.html file that is installed with the plugin.

        Installing the Outlook Client From Source

        *************** *** 67,73 **** the win32all installer. Finally, double-click the addin.py script in the Outlook2000 folder of the Spambayes distribution and you should be good to ! go.

        !

        Other Mail Clients

        If you use any other mail client on Windows, you currently need to --- 65,71 ---- the win32all installer. Finally, double-click the addin.py script in the Outlook2000 folder of the Spambayes distribution and you should be good to ! go. See the README.txt file in the Outlook2000 directory for more details

        !

        Non Outlook Solutions

        If you use any other mail client on Windows, you currently need to From sjoerd at users.sourceforge.net Mon Sep 8 05:09:27 2003 From: sjoerd at users.sourceforge.net (Sjoerd Mullender) Date: Mon Sep 8 07:09:31 2003 Subject: [Spambayes-checkins] spambayes/scripts sb_imapfilter.py,1.4,1.5 Message-ID: Update of /cvsroot/spambayes/spambayes/scripts In directory sc8-pr-cvs1:/tmp/cvs-serv25754/scripts Modified Files: sb_imapfilter.py Log Message: Count all messages being classified instead of just the ones from the last folder. Index: sb_imapfilter.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/scripts/sb_imapfilter.py,v retrieving revision 1.4 retrieving revision 1.5 diff -C2 -d -r1.4 -r1.5 *** sb_imapfilter.py 8 Sep 2003 09:19:44 -0000 1.4 --- sb_imapfilter.py 8 Sep 2003 11:09:25 -0000 1.5 *************** *** 644,648 **** if options["globals", "verbose"]: t = time.time() ! count = None # Select the spam folder and unsure folder to make sure they exist --- 644,651 ---- if options["globals", "verbose"]: t = time.time() ! count = {} ! count["ham"] = 0 ! count["spam"] = 0 ! count["unsure"] = 0 # Select the spam folder and unsure folder to make sure they exist *************** *** 654,659 **** imap.SelectFolder(filter_folder) folder = IMAPFolder(filter_folder) ! count = folder.Filter(self.classifier, self.spam_folder, self.unsure_folder) if options["globals", "verbose"]: --- 657,664 ---- imap.SelectFolder(filter_folder) folder = IMAPFolder(filter_folder) ! subcount = folder.Filter(self.classifier, self.spam_folder, self.unsure_folder) + for key in count.keys(): + count[key] += subcount.get(key, 0) if options["globals", "verbose"]: From sjoerd at acm.org Mon Sep 8 14:09:19 2003 From: sjoerd at acm.org (Sjoerd Mullender) Date: Mon Sep 8 07:09:36 2003 Subject: [Spambayes-checkins] spambayes/scripts sb_imapfilter.py,1.3,1.4 In-Reply-To: <1ED4ECF91CDED24C8D012BCF2B034F130212AE53@its-xchg4.massey.ac.nz> References: <1ED4ECF91CDED24C8D012BCF2B034F130212AE53@its-xchg4.massey.ac.nz> Message-ID: <20030908110919.BD41A74132@indus.ins.cwi.nl> On Mon, Sep 8 2003 "Tony Meyer" wrote: > > Trivial fix for IMAP over SSL. > [...] > > ! from imaplib import IMAP_SSL as BaseIMAP > [...] > > ! from imaplib import IMAP4_SSL as BaseIMAP > > . Thanks. (it does say, "untested" ) With this fix it does seem to be working. However, I do see that messages get scored every time I run sb_imapfilter -c. Another checkin to improve things is on its way. -- Sjoerd Mullender From mhammond at users.sourceforge.net Mon Sep 8 06:01:46 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Mon Sep 8 08:01:50 2003 Subject: [Spambayes-checkins] website quotes.ht,1.5,1.6 Message-ID: Update of /cvsroot/spambayes/website In directory sc8-pr-cvs1:/tmp/cvs-serv3027 Modified Files: quotes.ht Log Message: You would think I could spell my own name correctly Index: quotes.ht =================================================================== RCS file: /cvsroot/spambayes/website/quotes.ht,v retrieving revision 1.5 retrieving revision 1.6 diff -C2 -d -r1.5 -r1.6 *** quotes.ht 4 Sep 2003 23:29:47 -0000 1.5 --- quotes.ht 8 Sep 2003 12:01:44 -0000 1.6 *************** *** 79,83 ****

        I followed up too :) I have a working Outlook Addin in place.
        ! Mark Hammod         uttering words he is likely to regret for years to come

        --- 79,83 ----

        I followed up too :) I have a working Outlook Addin in place.
        ! Mark Hammond uttering words he is likely to regret for years to come

        From sjoerd at users.sourceforge.net Mon Sep 8 08:46:59 2003 From: sjoerd at users.sourceforge.net (Sjoerd Mullender) Date: Mon Sep 8 10:47:06 2003 Subject: [Spambayes-checkins] spambayes/spambayes message.py,1.35,1.36 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv8250 Modified Files: message.py Log Message: Use comparison with None for selve instance, since a shelve instance can compare false if the shelve is empty. I believe this fixes bug 801952 "Imapfilter appending headers". Index: message.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/message.py,v retrieving revision 1.35 retrieving revision 1.36 diff -C2 -d -r1.35 -r1.36 *** message.py 5 Sep 2003 01:25:27 -0000 1.35 --- message.py 8 Sep 2003 14:46:57 -0000 1.36 *************** *** 123,131 **** def store(self): ! if self.db: self.db.sync() def _getState(self, msg): ! if self.db: try: (msg.c, msg.t) = self.db[msg.getId()] --- 123,131 ---- def store(self): ! if self.db is not None: self.db.sync() def _getState(self, msg): ! if self.db is not None: try: (msg.c, msg.t) = self.db[msg.getId()] *************** *** 134,142 **** def _setState(self, msg): ! if self.db: self.db[msg.getId()] = (msg.c, msg.t) def _delState(self, msg): ! if self.db: del self.db[msg.getId()] --- 134,142 ---- def _setState(self, msg): ! if self.db is not None: self.db[msg.getId()] = (msg.c, msg.t) def _delState(self, msg): ! if self.db is not None: del self.db[msg.getId()] *************** *** 273,278 **** def addSBHeaders(self, prob, clues): ! '''Add hammie header, and remember message's classification. Also, ! add optional headers if needed.''' if prob < options['Categorization','ham_cutoff']: --- 273,278 ---- def addSBHeaders(self, prob, clues): ! """Add hammie header, and remember message's classification. Also, ! add optional headers if needed.""" if prob < options['Categorization','ham_cutoff']: From mhammond at users.sourceforge.net Mon Sep 8 21:08:45 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Tue Sep 9 01:12:07 2003 Subject: [Spambayes-checkins] website windows.ht,1.27,1.28 Message-ID: Update of /cvsroot/spambayes/website In directory sc8-pr-cvs1:/tmp/cvs-serv6709 Modified Files: windows.ht Log Message: 008.1 Index: windows.ht =================================================================== RCS file: /cvsroot/spambayes/website/windows.ht,v retrieving revision 1.27 retrieving revision 1.28 diff -C2 -d -r1.27 -r1.28 *** windows.ht 8 Sep 2003 10:55:58 -0000 1.27 --- windows.ht 9 Sep 2003 03:08:42 -0000 1.28 *************** *** 11,17 ****

        Latest Release

        !

        The latest release is Version 008 - see the release notes ! or download the installation program.

        --- 11,17 ----

        Latest Release

        !

        The latest release is Version 008.1 - see the release notes ! or download the installation program.

        From mhammond at users.sourceforge.net Mon Sep 8 18:59:08 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Tue Sep 9 01:16:44 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs FolderSelector.py, 1.27, 1.28 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs In directory sc8-pr-cvs1:/tmp/cvs-serv19723 Modified Files: FolderSelector.py Log Message: Handle message store exceptions when populating the list. Index: FolderSelector.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/FolderSelector.py,v retrieving revision 1.27 retrieving revision 1.28 diff -C2 -d -r1.27 -r1.28 *** FolderSelector.py 5 Sep 2003 06:54:23 -0000 1.27 --- FolderSelector.py 9 Sep 2003 00:59:06 -0000 1.28 *************** *** 266,271 **** # Compare the eid of the stores, then the objects CompareEntryIDs = self.manager.message_store.session.CompareEntryIDs ! return CompareEntryIDs(mapi.BinFromHex(id1[0]), mapi.BinFromHex(id2[0])) and \ ! CompareEntryIDs(mapi.BinFromHex(id1[1]), mapi.BinFromHex(id2[1])) def InIDs(self, id, ids): --- 266,275 ---- # Compare the eid of the stores, then the objects CompareEntryIDs = self.manager.message_store.session.CompareEntryIDs ! try: ! return CompareEntryIDs(mapi.BinFromHex(id1[0]), mapi.BinFromHex(id2[0])) and \ ! CompareEntryIDs(mapi.BinFromHex(id1[1]), mapi.BinFromHex(id2[1])) ! except pythoncom.com_error: ! # invalid IDs are never the same ! return False def InIDs(self, id, ids): *************** *** 339,343 **** folders_to_expand = [] for folder_id in self.selected_ids: ! folder = self.manager.message_store.GetFolder(folder_id) while folder is not None: parent = folder.GetParent() --- 343,351 ---- folders_to_expand = [] for folder_id in self.selected_ids: ! try: ! folder = self.manager.message_store.GetFolder(folder_id) ! except self.manager.message_store.MsgStoreException, details: ! print "Can't find a folder to expand:", details ! folder = None while folder is not None: parent = folder.GetParent() *************** *** 636,640 **** mgr.dialog_parser = dialogs.LoadDialogs() ! ids = [] d=FolderSelector(0, mgr, ids, single_select = False) if d.DoModal() != win32con.IDOK: --- 644,648 ---- mgr.dialog_parser = dialogs.LoadDialogs() ! ids = [("0000","0000"),] # invalid ID for testing. d=FolderSelector(0, mgr, ids, single_select = False) if d.DoModal() != win32con.IDOK: From mhammond at users.sourceforge.net Mon Sep 8 20:14:31 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Tue Sep 9 01:17:17 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs FolderSelector.py, 1.29, 1.30 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs In directory sc8-pr-cvs1:/tmp/cvs-serv31757 Modified Files: FolderSelector.py Log Message: Get the "New Folder" button working again (*sigh*) Index: FolderSelector.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/FolderSelector.py,v retrieving revision 1.29 retrieving revision 1.30 diff -C2 -d -r1.29 -r1.30 *** FolderSelector.py 9 Sep 2003 01:26:12 -0000 1.29 --- FolderSelector.py 9 Sep 2003 02:14:28 -0000 1.30 *************** *** 194,205 **** return array.array("c", buf), extra def UnpackTVItem(buffer): item_mask, item_hItem, item_state, item_stateMask, \ item_textptr, item_cchText, item_image, item_selimage, \ item_cChildren, item_param = struct.unpack("10i", buffer) ! # ensure only items listed by the mask are valid if not (item_mask & commctrl.TVIF_TEXT): item_textptr = item_cchText = None if not (item_mask & commctrl.TVIF_CHILDREN): item_cChildren = None - if not (item_mask & commctrl.TVIF_HANDLE): item_hItem = None if not (item_mask & commctrl.TVIF_IMAGE): item_image = None if not (item_mask & commctrl.TVIF_PARAM): item_param = None --- 194,228 ---- return array.array("c", buf), extra + # Make a new buffer suitable for querying hitem's attributes. + def EmptyTVITEM(hitem, mask = None, text_buf_size=512): + extra = [] # objects we must keep references to + if mask is None: + mask = commctrl.TVIF_HANDLE | commctrl.TVIF_STATE | commctrl.TVIF_TEXT | \ + commctrl.TVIF_IMAGE | commctrl.TVIF_SELECTEDIMAGE | \ + commctrl.TVIF_CHILDREN | commctrl.TVIF_PARAM + if mask & commctrl.TVIF_TEXT: + text_buffer = array.array("c", "\0" * text_buf_size) + extra.append(text_buffer) + text_addr, text_len = text_buffer.buffer_info() + else: + text_addr = text_len = 0 + format = "iiiiiiiiii" + buf = struct.pack(format, + mask, hitem, + 0, 0, + text_addr, text_len, # text + 0, 0, + 0, 0) + return array.array("c", buf), extra + def UnpackTVItem(buffer): item_mask, item_hItem, item_state, item_stateMask, \ item_textptr, item_cchText, item_image, item_selimage, \ item_cChildren, item_param = struct.unpack("10i", buffer) ! # ensure only items listed by the mask are valid (except we assume the ! # handle is always valid - some notifications (eg, TVN_ENDLABELEDIT) set a ! # mask that doesn't include the handle, but the docs explicity say it is.) if not (item_mask & commctrl.TVIF_TEXT): item_textptr = item_cchText = None if not (item_mask & commctrl.TVIF_CHILDREN): item_cChildren = None if not (item_mask & commctrl.TVIF_IMAGE): item_image = None if not (item_mask & commctrl.TVIF_PARAM): item_param = None *************** *** 359,364 **** def _GetTVItem(self, h): ! text_buffer = "\0" * 1024 ! buffer, extra = PackTVITEM(h, 0, 0, text_buffer, None, None, None, -1) win32gui.SendMessage(self.list, commctrl.TVM_GETITEM, 0, buffer.buffer_info()[0]) --- 382,386 ---- def _GetTVItem(self, h): ! buffer, extra = EmptyTVITEM(h) win32gui.SendMessage(self.list, commctrl.TVM_GETITEM, 0, buffer.buffer_info()[0]) From xenogeist at users.sourceforge.net Mon Sep 8 19:07:28 2003 From: xenogeist at users.sourceforge.net (Adam Walker) Date: Tue Sep 9 01:18:21 2003 Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py,1.5,1.6 Message-ID: Update of /cvsroot/spambayes/spambayes/windows In directory sc8-pr-cvs1:/tmp/cvs-serv21080/windows Modified Files: pop3proxy_tray.py Log Message: Update the tray program to use the new script names. Index: pop3proxy_tray.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_tray.py,v retrieving revision 1.5 retrieving revision 1.6 diff -C2 -d -r1.5 -r1.6 *** pop3proxy_tray.py 2 Sep 2003 20:12:13 -0000 1.5 --- pop3proxy_tray.py 9 Sep 2003 01:07:26 -0000 1.6 *************** *** 33,38 **** sys.path.insert(-1, os.getcwd()) sys.path.insert(-1, os.path.dirname(os.getcwd())) ! import pop3proxy from spambayes import Dibbler from spambayes.Options import options --- 33,39 ---- sys.path.insert(-1, os.getcwd()) sys.path.insert(-1, os.path.dirname(os.getcwd())) + sys.path.insert(-1, os.path.join(os.path.dirname(os.getcwd()),"scripts")) ! import sb_server from spambayes import Dibbler from spambayes.Options import options *************** *** 73,83 **** # Get the custom icon ! startedIconPathName = "%s\\windows\\resources\\sb-started.ico" % \ ! (os.path.dirname(pop3proxy.__file__),) ! stoppedIconPathName = "%s\\windows\\resources\\sb-stopped.ico" % \ ! (os.path.dirname(pop3proxy.__file__),) # When 1.0a6 is released, the above line will need to change to: ## iconPathName = "%s\\..\\windows\\resources\\sbicon.ico" % \ ! ## (os.path.dirname(pop3proxy.__file__),) if os.path.isfile(startedIconPathName) and os.path.isfile(stoppedIconPathName): icon_flags = win32con.LR_LOADFROMFILE | win32con.LR_DEFAULTSIZE --- 74,84 ---- # Get the custom icon ! startedIconPathName = "%s\\..\\windows\\resources\\sb-started.ico" % \ ! (os.path.dirname(sb_server.__file__),) ! stoppedIconPathName = "%s\\..\\windows\\resources\\sb-stopped.ico" % \ ! (os.path.dirname(sb_server.__file__),) # When 1.0a6 is released, the above line will need to change to: ## iconPathName = "%s\\..\\windows\\resources\\sbicon.ico" % \ ! ## (os.path.dirname(sb_server.__file__),) if os.path.isfile(startedIconPathName) and os.path.isfile(stoppedIconPathName): icon_flags = win32con.LR_LOADFROMFILE | win32con.LR_DEFAULTSIZE *************** *** 93,101 **** self.tip = None ! # Start up pop3proxy # XXX This needs to be finished off. # XXX This should determine if we are using the service, and if so ! # XXX start that, and if not kick pop3proxy off in a separate thread. ! pop3proxy.prepare(state=pop3proxy.state) self.StartStop() --- 94,102 ---- self.tip = None ! # Start up sb_server # XXX This needs to be finished off. # XXX This should determine if we are using the service, and if so ! # XXX start that, and if not kick sb_server off in a separate thread. ! sb_server.prepare(state=sb_server.state) self.StartStop() *************** *** 105,110 **** #%i spam %i unsure %i session %i active tip = "SpamBayes %i spam %i ham %i unsure %i sessions %i active" %\ ! (pop3proxy.state.numSpams, pop3proxy.state.numHams, pop3proxy.state.numUnsure, ! pop3proxy.state.totalSessions, pop3proxy.state.activeSessions) else: tip = "SpamBayes is not running" --- 106,111 ---- #%i spam %i unsure %i session %i active tip = "SpamBayes %i spam %i ham %i unsure %i sessions %i active" %\ ! (sb_server.state.numSpams, sb_server.state.numHams, sb_server.state.numUnsure, ! sb_server.state.totalSessions, sb_server.state.activeSessions) else: tip = "SpamBayes is not running" *************** *** 167,171 **** def OnExit(self): if self.started: ! pop3proxy.stop(pop3proxy.state) self.started = False DestroyWindow(self.hwnd) --- 168,172 ---- def OnExit(self): if self.started: ! sb_server.stop(sb_server.state) self.started = False DestroyWindow(self.hwnd) *************** *** 173,178 **** def StartProxyThread(self): ! args = (pop3proxy.state,) ! thread.start_new_thread(pop3proxy.start, args) self.started = True --- 174,179 ---- def StartProxyThread(self): ! args = (sb_server.state,) ! thread.start_new_thread(sb_server.start, args) self.started = True *************** *** 180,187 **** # XXX This needs to be finished off. # XXX This should determine if we are using the service, and if so ! # XXX start/stop that, and if not kick pop3proxy off in a separate # XXX thread, or stop the thread that was started. if self.started: ! pop3proxy.stop(pop3proxy.state) self.started = False self.control_functions[START_STOP_ID] = ("Start SpamBayes", --- 181,188 ---- # XXX This needs to be finished off. # XXX This should determine if we are using the service, and if so ! # XXX start/stop that, and if not kick sb_server off in a separate # XXX thread, or stop the thread that was started. if self.started: ! sb_server.stop(sb_server.state) self.started = False self.control_functions[START_STOP_ID] = ("Start SpamBayes", From mhammond at users.sourceforge.net Mon Sep 8 19:26:14 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Tue Sep 9 01:23:16 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs FolderSelector.py, 1.28, 1.29 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs In directory sc8-pr-cvs1:/tmp/cvs-serv23849 Modified Files: FolderSelector.py Log Message: Another place we used to check for GetFolder() returning None, which it never does any more. Still a couple of places that are unguarded, but they would have previously failed with a None return anyway, so are left alone. Index: FolderSelector.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/FolderSelector.py,v retrieving revision 1.28 retrieving revision 1.29 diff -C2 -d -r1.28 -r1.29 *** FolderSelector.py 9 Sep 2003 00:59:06 -0000 1.28 --- FolderSelector.py 9 Sep 2003 01:26:12 -0000 1.29 *************** *** 91,106 **** temp_id = mapi.HexFromBin(store_eid), mapi.HexFromBin(eid) try: child_folder = manager.message_store.GetFolder(temp_id) ! if child_folder is not None: ! spec = FolderSpec(child_folder.GetID(), name, folder_spec.ignore_eids) ! # If we have no children at all, indicate ! # the item is not expandable. ! table = child_folder.OpenEntry().GetHierarchyTable(0) ! if table.GetRowCount(0) == 0: ! spec.children = [] ! else: ! spec.children = None # Flag as "not yet built" ! children.append(spec) ! except pythoncom.com_error, details: # Users have reported failure here - it is not clear if the # entire tree is going to fail, or just this folder --- 91,108 ---- temp_id = mapi.HexFromBin(store_eid), mapi.HexFromBin(eid) try: + # may get MsgStoreException for GetFolder, or + # a mapi exception for the underlying MAPI stuff we then call. + # Either way, just skip it. child_folder = manager.message_store.GetFolder(temp_id) ! spec = FolderSpec(child_folder.GetID(), name, folder_spec.ignore_eids) ! # If we have no children at all, indicate ! # the item is not expandable. ! table = child_folder.OpenEntry().GetHierarchyTable(0) ! if table.GetRowCount(0) == 0: ! spec.children = [] ! else: ! spec.children = None # Flag as "not yet built" ! children.append(spec) ! except (pythoncom.com_error, manager.message_store.MsgStoreException), details: # Users have reported failure here - it is not clear if the # entire tree is going to fail, or just this folder From xenogeist at users.sourceforge.net Mon Sep 8 19:26:18 2003 From: xenogeist at users.sourceforge.net (Adam Walker) Date: Tue Sep 9 01:23:24 2003 Subject: [Spambayes-checkins] spambayes/windows pop3proxy_service.py, 1.10, 1.11 Message-ID: Update of /cvsroot/spambayes/spambayes/windows In directory sc8-pr-cvs1:/tmp/cvs-serv24014/windows Modified Files: pop3proxy_service.py Log Message: Update the windows service to use the new script names. Index: pop3proxy_service.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_service.py,v retrieving revision 1.10 retrieving revision 1.11 diff -C2 -d -r1.10 -r1.11 *** pop3proxy_service.py 31 Aug 2003 02:17:46 -0000 1.10 --- pop3proxy_service.py 9 Sep 2003 01:26:16 -0000 1.11 *************** *** 1,3 **** ! # Run the pop3proxy as a WinNT service. Should work on Windows 2000 # and Windows XP. # --- 1,3 ---- ! # Run the sb_server as a WinNT service. Should work on Windows 2000 # and Windows XP. # *************** *** 26,29 **** --- 26,31 ---- this_filename=__file__ except NameError: + this_filename = sys.argv[0] + if not os.path.isabs(sys.argv[0]): # Python 2.3 __main__ # patch up sys.argv, as our cwd will confuse service registration code *************** *** 31,37 **** --- 33,42 ---- this_filename = sys.argv[0] + print sys.argv[0] sb_dir = os.path.dirname(os.path.dirname(this_filename)) + sb_scripts_dir = os.path.join(sb_dir,"scripts") sys.path.insert(0, sb_dir) + sys.path.insert(-1, sb_scripts_dir) # and change directory here, so pop3proxy uses the default # config file etc *************** *** 44,48 **** # The spambayes imports we need. ! import pop3proxy # The win32 specific modules. --- 49,53 ---- # The spambayes imports we need. ! import sb_server # The win32 specific modules. *************** *** 68,73 **** class Service(win32serviceutil.ServiceFramework): _svc_name_ = "pop3proxy" ! _svc_display_name_ = "SpamBayes pop3proxy Service" _svc_deps_ = ['tcpip'] # We depend on the tcpip service. def __init__(self, args): --- 73,80 ---- class Service(win32serviceutil.ServiceFramework): + # The script name was changed to "sb_server" but I'll leave this as pop3proxy + # overwise people might accidently run two proxies. _svc_name_ = "pop3proxy" ! _svc_display_name_ = "SpamBayes Service" _svc_deps_ = ['tcpip'] # We depend on the tcpip service. def __init__(self, args): *************** *** 80,90 **** self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING) self.event_stopping.set() ! pop3proxy.stop(pop3proxy.state) def SvcDoRun(self): # Setup our state etc ! pop3proxy.prepare(state=pop3proxy.state) ! assert not pop3proxy.state.launchUI, "Service can't launch a UI" # Start the thread running the server. --- 87,97 ---- self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING) self.event_stopping.set() ! sb_server.stop(sb_server.state) def SvcDoRun(self): # Setup our state etc ! sb_server.prepare(state=sb_server.state) ! assert not sb_server.state.launchUI, "Service can't launch a UI" # Start the thread running the server. *************** *** 122,126 **** # Write another event log record. ! s = pop3proxy.state status = " after %d sessions (%d ham, %d spam, %d unsure)" % \ (s.totalSessions, s.numHams, s.numSpams, s.numUnsure) --- 129,133 ---- # Write another event log record. ! s = sb_server.state status = " after %d sessions (%d ham, %d spam, %d unsure)" % \ (s.totalSessions, s.numHams, s.numSpams, s.numUnsure) *************** *** 135,139 **** try: try: ! pop3proxy.start(pop3proxy.state) except SystemExit: # user requested shutdown --- 142,146 ---- try: try: ! sb_server.start(sb_server.state) except SystemExit: # user requested shutdown From mhammond at users.sourceforge.net Mon Sep 8 21:09:51 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Tue Sep 9 01:23:33 2003 Subject: [Spambayes-checkins] spambayes/spambayes Version.py,1.19,1.20 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv6900 Modified Files: Version.py Log Message: 0.81 of Outlook. Index: Version.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/Version.py,v retrieving revision 1.19 retrieving revision 1.20 diff -C2 -d -r1.19 -r1.20 *** Version.py 8 Sep 2003 07:56:50 -0000 1.19 --- Version.py 9 Sep 2003 03:09:49 -0000 1.20 *************** *** 31,38 **** }, "Outlook" : { ! "Version": 0.8, ! "BinaryVersion": 0.8, "Description": "SpamBayes Outlook Addin", ! "Date": "September 8, 2003", "Full Description": "%(Description)s, version %(Version)s (%(Date)s)", "Full Description Binary": --- 31,38 ---- }, "Outlook" : { ! "Version": 0.81, ! "BinaryVersion": 0.81, "Description": "SpamBayes Outlook Addin", ! "Date": "September 9, 2003", "Full Description": "%(Description)s, version %(Version)s (%(Date)s)", "Full Description Binary": From mhammond at users.sourceforge.net Mon Sep 8 19:13:10 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Tue Sep 9 01:23:38 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs opt_processors.py, 1.13, 1.14 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs In directory sc8-pr-cvs1:/tmp/cvs-serv21813 Modified Files: opt_processors.py Log Message: Handle message store exception in the folder ID processor. Index: opt_processors.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/opt_processors.py,v retrieving revision 1.13 retrieving revision 1.14 diff -C2 -d -r1.13 -r1.14 *** opt_processors.py 4 Sep 2003 11:59:05 -0000 1.13 --- opt_processors.py 9 Sep 2003 01:13:07 -0000 1.14 *************** *** 318,329 **** for eid in ids: if eid is not None: ! folder = mgr.message_store.GetFolder(eid) ! if folder is None: ! name = "" ! else: if self.use_fqn: name = folder.GetFQName() else: name = folder.name names.append(name) win32gui.SetWindowText(self.GetControl(), self.name_joiner.join(names)) --- 318,329 ---- for eid in ids: if eid is not None: ! try: ! folder = mgr.message_store.GetFolder(eid) if self.use_fqn: name = folder.GetFQName() else: name = folder.name + except mgr.message_store.MsgStoreException: + name = "" names.append(name) win32gui.SetWindowText(self.GetControl(), self.name_joiner.join(names)) From anadelonbrin at users.sourceforge.net Mon Sep 8 18:36:57 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Tue Sep 9 01:34:31 2003 Subject: [Spambayes-checkins] spambayes/scripts sb_imapfilter.py,1.5,1.6 Message-ID: Update of /cvsroot/spambayes/spambayes/scripts In directory sc8-pr-cvs1:/tmp/cvs-serv15856/scripts Modified Files: sb_imapfilter.py Log Message: Another bug found by Rafael Scholl - some imap servers don't put the quotation mark around folder names like they should (according to the RFC), and we stripped them off whether they were there or not, which meant that some names would be missing the first and last characters in the folder list. Fix this. Index: sb_imapfilter.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/scripts/sb_imapfilter.py,v retrieving revision 1.5 retrieving revision 1.6 diff -C2 -d -r1.5 -r1.6 *** sb_imapfilter.py 8 Sep 2003 11:09:25 -0000 1.5 --- sb_imapfilter.py 9 Sep 2003 00:36:54 -0000 1.6 *************** *** 268,274 **** # the case if self.folder_delimiter == ',': ! print """WARNING: Your imap server uses commas as the folder ! delimiter. This may cause unpredictable errors.""" ! folders.append(fol[m.end()+5:-1]) folders.sort() return folders --- 268,275 ---- # the case if self.folder_delimiter == ',': ! print "WARNING: Your imap server uses a comma as the " \ ! "folder delimiter. This may cause unpredictable " \ ! "errors." ! folders.append(fol[m.end()+4:].strip('"')) folders.sort() return folders From anadelonbrin at users.sourceforge.net Tue Sep 9 00:05:17 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Tue Sep 9 02:05:21 2003 Subject: [Spambayes-checkins] spambayes/spambayes ProxyUI.py,1.20,1.21 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv27936/spambayes Modified Files: ProxyUI.py Log Message: A bug report had 'text' ending up as None, which stoped the messages displaying. This handles that error, so that things keep working, at least, but it would be good to figure out how it is that 'text' can become None. As far as I can see, it means that get_payload() is returning None, but I don't think it's meant to be able to do that. Index: ProxyUI.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/ProxyUI.py,v retrieving revision 1.20 retrieving revision 1.21 diff -C2 -d -r1.20 -r1.21 *** ProxyUI.py 1 Sep 2003 06:07:27 -0000 1.20 --- ProxyUI.py 9 Sep 2003 06:05:15 -0000 1.21 *************** *** 489,492 **** --- 489,494 ---- if type(text) == type([]): # gotta be a 'right' way to do this text = "(this message is a digest of %s messages)" % (len(text)) + elif text is None: + text = "(this message has no body)" else: text = text.replace(' ', ' ') # Else they'll be quoted From mhammond at users.sourceforge.net Tue Sep 9 00:17:36 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Tue Sep 9 02:17:39 2003 Subject: [Spambayes-checkins] spambayes/spambayes storage.py,1.28,1.29 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv29384 Modified Files: storage.py Log Message: Add a close method to the various storage classes. Index: storage.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v retrieving revision 1.28 retrieving revision 1.29 diff -C2 -d -r1.28 -r1.29 *** storage.py 5 Sep 2003 01:15:28 -0000 1.28 --- storage.py 9 Sep 2003 06:17:33 -0000 1.29 *************** *** 141,144 **** --- 141,148 ---- fp.close() + def close(self): + # we keep no reasources open - nothing to do + pass + # Values for our changed words map WORD_DELETED = "D" *************** *** 157,160 **** --- 161,172 ---- self.load() + def close(self): + # Close our underlying database. Better not assume all databases + # have close functions! + def noop(): pass + getattr(self.db, "close", noop)() + getattr(self.dbm, "close", noop)() + # should not be a need to drop the 'dbm' or 'db' attributes. + def load(self): '''Load state from database''' *************** *** 292,295 **** --- 304,313 ---- self.db_name = db_name self.load() + + def close(self): + '''Release all database resources''' + # As we (presumably) aren't as constrained as we are by file locking, + # don't force sub-classes to override + pass def load(self): From anadelonbrin at users.sourceforge.net Tue Sep 9 00:51:02 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Tue Sep 9 02:51:06 2003 Subject: [Spambayes-checkins] spambayes/scripts sb_server.py,1.1,1.2 Message-ID: Update of /cvsroot/spambayes/spambayes/scripts In directory sc8-pr-cvs1:/tmp/cvs-serv1951/scripts Modified Files: sb_server.py Log Message: sb_server should store and close the db before reopening it. gdbm fails if we do not do this. Index: sb_server.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/scripts/sb_server.py,v retrieving revision 1.1 retrieving revision 1.2 diff -C2 -d -r1.1 -r1.2 *** sb_server.py 5 Sep 2003 01:16:45 -0000 1.1 --- sb_server.py 9 Sep 2003 06:51:00 -0000 1.2 *************** *** 718,721 **** --- 718,726 ---- del proxyListeners[:] + # Close the database; we should anyway, and gdbm complains if we + # try to reopen it without closing it first. + state.bayes.store() + state.bayes.close() + prepare(state) _createProxies(state.servers, state.proxyPorts) From anadelonbrin at users.sourceforge.net Tue Sep 9 01:03:57 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Tue Sep 9 03:03:59 2003 Subject: [Spambayes-checkins] spambayes/spambayes Options.py,1.71,1.72 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv6536/spambayes Modified Files: Options.py Log Message: Implement half of [ 801699 ] migration from hammie to sb_* scripts (although in a different way) This makes it optional to write the X-Spambayes-Trained header, to ease transition from hammie.py Index: Options.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v retrieving revision 1.71 retrieving revision 1.72 diff -C2 -d -r1.71 -r1.72 *** Options.py 6 Sep 2003 04:03:35 -0000 1.71 --- Options.py 9 Sep 2003 07:03:54 -0000 1.72 *************** *** 569,572 **** --- 569,579 ---- HEADER_NAME, RESTORE), + ("include_trained", "Add trained header", True, + """sb_mboxtrain.py can add a header that details how a message was + trained, which lets you keep track of it, and appropriately + re-train messages. However, if you would rather mboxtrain didn't + rewrite the message files, you can disable this option.""", + BOOLEAN, RESTORE), + ("trained_header_name", "Trained header name", "X-Spambayes-Trained", """When training on a message, the name of the header to add with how From anadelonbrin at users.sourceforge.net Tue Sep 9 01:03:57 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Tue Sep 9 03:04:04 2003 Subject: [Spambayes-checkins] spambayes/scripts sb_mboxtrain.py,1.1,1.2 Message-ID: Update of /cvsroot/spambayes/spambayes/scripts In directory sc8-pr-cvs1:/tmp/cvs-serv6536/scripts Modified Files: sb_mboxtrain.py Log Message: Implement half of [ 801699 ] migration from hammie to sb_* scripts (although in a different way) This makes it optional to write the X-Spambayes-Trained header, to ease transition from hammie.py Index: sb_mboxtrain.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/scripts/sb_mboxtrain.py,v retrieving revision 1.1 retrieving revision 1.2 diff -C2 -d -r1.1 -r1.2 *** sb_mboxtrain.py 5 Sep 2003 01:16:45 -0000 1.1 --- sb_mboxtrain.py 9 Sep 2003 07:03:54 -0000 1.2 *************** *** 116,119 **** --- 116,121 ---- continue trained += 1 + if not options["Headers", "include_trained"]: + continue f = file(tfn, "wb") f.write(msg.as_string()) *************** *** 155,177 **** if msg_train(h, msg, is_spam, force): trained += 1 ! # Write it out with the Unix "From " line ! outf.write(msg.as_string(True)) ! outf.seek(0) ! try: ! os.ftruncate(f.fileno(), 0) ! f.seek(0) ! except: ! # If anything goes wrong, don't try to write ! print "Problem truncating mbox--nothing written" ! raise ! try: ! for line in outf.xreadlines(): ! f.write(line) ! except: ! print >> sys.stderr ("Problem writing mbox! Sorry, " ! "I tried my best, but your mail " ! "may be corrupted.") ! raise fcntl.lockf(f, fcntl.LOCK_UN) f.close() --- 157,181 ---- if msg_train(h, msg, is_spam, force): trained += 1 ! if not options["Headers", "include_trained"]: ! # Write it out with the Unix "From " line ! outf.write(msg.as_string(True)) ! if not options["Headers", "include_trained"]: ! outf.seek(0) ! try: ! os.ftruncate(f.fileno(), 0) ! f.seek(0) ! except: ! # If anything goes wrong, don't try to write ! print "Problem truncating mbox--nothing written" ! raise ! try: ! for line in outf.xreadlines(): ! f.write(line) ! except: ! print >> sys.stderr ("Problem writing mbox! Sorry, " ! "I tried my best, but your mail " ! "may be corrupted.") ! raise fcntl.lockf(f, fcntl.LOCK_UN) f.close() *************** *** 203,206 **** --- 207,212 ---- msg_train(h, msg, is_spam, force) trained += 1 + if not options["Headers", "include_trained"]: + continue f = file(tfn, "wb") f.write(msg.as_string()) *************** *** 290,293 **** --- 296,300 ---- if loud: print "Training ham (%s):" % g train(h, g, False, force, trainnew, removetrained) + sys.stdout.flush() save = True *************** *** 295,298 **** --- 302,306 ---- if loud: print "Training spam (%s):" % s train(h, s, True, force, trainnew, removetrained) + sys.stdout.flush() save = True From anadelonbrin at users.sourceforge.net Tue Sep 9 01:44:01 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Tue Sep 9 03:44:06 2003 Subject: [Spambayes-checkins] spambayes README.txt,1.51,1.52 Message-ID: Update of /cvsroot/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv12808 Modified Files: README.txt Log Message: Update documentation to new script names. Procmail section needs finishing. Index: README.txt =================================================================== RCS file: /cvsroot/spambayes/spambayes/README.txt,v retrieving revision 1.51 retrieving revision 1.52 diff -C2 -d -r1.51 -r1.52 *** README.txt 25 Aug 2003 08:55:19 -0000 1.51 --- README.txt 9 Sep 2003 07:43:59 -0000 1.52 *************** *** 67,71 **** If you get your mail from a POP3 server, then all you should need to do to get running is change your mail client to send and receive mail from ! "localhost", and then run "python pop3proxy.py -b" in the directory you expanded the SpamBayes source into. This will open a web browser window - click the "Configuration" link at the top right and fill in the various --- 67,71 ---- If you get your mail from a POP3 server, then all you should need to do to get running is change your mail client to send and receive mail from ! "localhost", and then run "python sb_server.py -b" in the directory you expanded the SpamBayes source into. This will open a web browser window - click the "Configuration" link at the top right and fill in the various *************** *** 113,117 **** Now launch pop3proxy, either by running the "pop3proxy_service.py" script (for those using Windows 2000, Windows NT or Windows XP), or the ! "pop3proxy.py" script (for everyone else). All you need to do to configure SpamBayes is to open a web page to --- 113,117 ---- Now launch pop3proxy, either by running the "pop3proxy_service.py" script (for those using Windows 2000, Windows NT or Windows XP), or the ! "sb_server.py" script (for everyone else). All you need to do to configure SpamBayes is to open a web page to *************** *** 138,145 **** ----------- ! To configure SpamBayes, run the "imapfilter.py" script, and open a web page ! to , click on the "Configuration" link at the top ! right, and fill in the relevant details. Everything should be ok with the ! defaults, except for the server information at the top. You now need to let SpamBayes know which IMAP folders it should work with. --- 138,145 ---- ----------- ! To configure SpamBayes, run the "sb_imapfilter.py" script, and open a web ! page to , click on the "Configuration" link at the ! top right, and fill in the relevant details. Everything should be ok with ! the defaults, except for the server information at the top. You now need to let SpamBayes know which IMAP folders it should work with. *************** *** 154,158 **** You then need to set the IMAP filter up to run periodically. At the moment, you'll need to do this from a command (or DOS) prompt. You should run the ! command "python imapfilter.py -c -t -l 5". The '-c' means that the script should classify new mail, the '-t' means that the script should train any mail that you have told it to, and the '-l 5' means that the script should --- 154,158 ---- You then need to set the IMAP filter up to run periodically. At the moment, you'll need to do this from a command (or DOS) prompt. You should run the ! command "python sb_imapfilter.py -c -t -l 5". The '-c' means that the script should classify new mail, the '-t' means that the script should train any mail that you have told it to, and the '-l 5' means that the script should *************** *** 167,171 **** Procmail is straightforward. ! First, create a SpamBayes database, by running "hammiefilter.py -n". If you have some mail around that you can use to train it, do you (see the "command line training" section below). Note that if you don't, all your --- 167,171 ---- Procmail is straightforward. ! First, create a SpamBayes database, by running "sb_filter.py -n". If you have some mail around that you can use to train it, do you (see the "command line training" section below). Note that if you don't, all your *************** *** 187,191 **** :0 fw:hamlock ! | /usr/local/bin/hammie.py -f -d -p $HOME/hammie.db The above Procmail recipe tells it to run /usr/local/bin/hammie.py in filter --- 187,191 ---- :0 fw:hamlock ! | /usr/local/bin/hammie.py -f -d -p $HOME/.hammie.db The above Procmail recipe tells it to run /usr/local/bin/hammie.py in filter *************** *** 199,203 **** The result of running hammie.py in filter mode is that Procmail will use the output from the run as the mail message for further processing downstream. ! Hammie.py inserts an X-SpamBayes-Classification header in the output message which looks like: --- 199,203 ---- The result of running hammie.py in filter mode is that Procmail will use the output from the run as the mail message for further processing downstream. ! hammie.py inserts an X-SpamBayes-Classification header in the output message which looks like: From anadelonbrin at users.sourceforge.net Tue Sep 9 01:53:58 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Tue Sep 9 03:54:02 2003 Subject: [Spambayes-checkins] spambayes/spambayes OptionsClass.py,1.12,1.13 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv14538/spambayes Modified Files: OptionsClass.py Log Message: Fix for [ 802347 ] multiline options saved incorrectly Index: OptionsClass.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/OptionsClass.py,v retrieving revision 1.12 retrieving revision 1.13 diff -C2 -d -r1.12 -r1.13 *** OptionsClass.py 5 Sep 2003 01:15:28 -0000 1.12 --- OptionsClass.py 9 Sep 2003 07:53:56 -0000 1.13 *************** *** 436,440 **** out.write(optname) out.write(vi) ! out.write(self.unconvert(sectname, optname)) out.write('\n') written.append((sectname, optname)) --- 436,441 ---- out.write(optname) out.write(vi) ! newval = self.unconvert(sectname, optname) ! out.write(newval.replace("\n", "\n\t")) out.write('\n') written.append((sectname, optname)) *************** *** 465,469 **** out.write(opt) out.write(vi) ! out.write(self.unconvert(sect, opt)) out.write('\n') written.append((sect, opt)) --- 466,471 ---- out.write(opt) out.write(vi) ! newval = self.unconvert(sect, opt) ! out.write(newval.replace("\n", "\n\t")) out.write('\n') written.append((sect, opt)) From anadelonbrin at users.sourceforge.net Tue Sep 9 02:07:12 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Tue Sep 9 04:07:18 2003 Subject: [Spambayes-checkins] spambayes/scripts sb_imapfilter.py,1.6,1.7 Message-ID: Update of /cvsroot/spambayes/spambayes/scripts In directory sc8-pr-cvs1:/tmp/cvs-serv16641/scripts Modified Files: sb_imapfilter.py Log Message: Fix for [ 802545 ] crash when loggin off imapfilter UI Index: sb_imapfilter.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/scripts/sb_imapfilter.py,v retrieving revision 1.6 retrieving revision 1.7 diff -C2 -d -r1.6 -r1.7 *** sb_imapfilter.py 9 Sep 2003 00:36:54 -0000 1.6 --- sb_imapfilter.py 9 Sep 2003 08:07:10 -0000 1.7 *************** *** 199,202 **** --- 199,203 ---- self.current_folder = None self.do_expunge = do_expunge + self.logged_in = False def login(self, username, pwd): *************** *** 210,218 **** else: raise def logout(self): # sign off if self.do_expunge: ! self.expunge() BaseIMAP.logout(self) # superclass logout --- 211,232 ---- else: raise + self.logged_in = True def logout(self): # sign off if self.do_expunge: ! # we may never have logged in, in which case we do nothing ! if self.logged_in: ! # expunge messages from the spam and unsure folders ! for fol in ["spam_folder", ! "unsure_folder",]: ! self.select(options["imap", fol]) ! self.expunge() ! # expunge messages from the ham and spam training folders ! for fol_list in ["ham_train_folders", ! "spam_train_folders",]: ! for fol in options["imap", fol_list]: ! self.select(fol) ! self.expunge() BaseIMAP.logout(self) # superclass logout From anadelonbrin at users.sourceforge.net Tue Sep 9 02:21:21 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Tue Sep 9 04:21:33 2003 Subject: [Spambayes-checkins] spambayes CHANGELOG.txt,1.17,1.18 Message-ID: Update of /cvsroot/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv18743 Modified Files: CHANGELOG.txt Log Message: Bring up to date. Index: CHANGELOG.txt =================================================================== RCS file: /cvsroot/spambayes/spambayes/CHANGELOG.txt,v retrieving revision 1.17 retrieving revision 1.18 diff -C2 -d -r1.17 -r1.18 *** CHANGELOG.txt 1 Sep 2003 21:36:42 -0000 1.17 --- CHANGELOG.txt 9 Sep 2003 08:21:19 -0000 1.18 *************** *** 1,6 **** --- 1,29 ---- [Note that all dates are in English, not American format - i.e. day/month/year] + Alpha Release 6 + =============== + Tony Meyer 09/09/2003 Fix for [ 802545 ] crash when loggin off imapfilter UI + Tony Meyer 09/09/2003 Fix for [ 802347 ] multiline options saved incorrectly + Tony Meyer 09/09/2003 Implement half of [ 801699 ] migration from hammie to sb_* scripts (although in a different way) + Tony Meyer 09/09/2003 sb_server should store and close the db before reopening it. gdbm fails if we do not do this. Fix this. + Tony Meyer 09/09/2003 imapfilter: correctly handle imap servers that (wrongly) fail to put folder names in quotation marks + Mark Hammond 09/09/2003 Add a close method to the various storage classes. + Sjoerd Mullender 08/09/2003 Fix for [ 801952 ] Imapfilter appending headers + Sjoerd Mullender 08/09/2003 imapfilter: Count all messages being classified instead of just the ones from the last folder. + Sjoerd Mullender 08/09/2003 Trivial fix for IMAP over SSL. + Tony Meyer 08/09/2003 imapfilter: handle a folder name as a literal when presenting a list to choose from + Tony Meyer 08/09/2003 imapfilter: handle imap servers that don't pass a blank result line for an empty search + Mark Hammond 08/09/2003 Outlook: When we fail to add the 'Spam' field to a read-only store (eg, hotmail), complain less loudly. + Mark Hammond 08/09/2003 Fix [ 798362 ] Toolbar becomes unresponsive and must be recreated + Tony Meyer 06/09/2003 Add a missing file that's necessary to try out the sb_pop3dnd.py script, and the version and options necessary. + Tony Meyer 06/09/2003 Fix [ 800555 ] 1.0a5 release missing key outlook files. + Tony Meyer 05/09/2003 Remove backwards compat code for options, and update all (I hope) the remaining code that uses it. + Tony Meyer 05/09/2003 Move and rename all top-level scripts other than setup.py + Alpha Release 5 =============== + Tony Meyer 03/09/2003 imapfilter: We would crash if we hadn't set anything up and didn't prompt for password. + Adam Walker 03/09/2003 pop3proxy_tray: Switch icons based on if the proxy is running or not. Provide some info found on the information page in the tooltip of the icon. + Tony Meyer 02/09/2003 Add a rough autoconfigure script, which will set up both spambayes and a mail client to use pop3proxy Richie Hindle 01/09/2003 Fix [ 797890 ] "assert hamcount <= nham" problem. Tony Meyer 01/09/2003 Fix [ spambayes-Bugs-796996 ] smtp server not started until restart. From anadelonbrin at users.sourceforge.net Tue Sep 9 02:55:53 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Tue Sep 9 04:55:57 2003 Subject: [Spambayes-checkins] spambayes/windows autoconfigure.py,1.4,1.5 Message-ID: Update of /cvsroot/spambayes/spambayes/windows In directory sc8-pr-cvs1:/tmp/cvs-serv24726/windows Modified Files: autoconfigure.py Log Message: Fix Outlook Express configuration to use the current user's settings, which removes the need to figure out which registry key to use. I accidently checked these in previously, but didn't notate them: add support for pocomail (thanks to Romain Guy) improve the move_to_next_port function to not use ports already taken by spambayes (thanks to Romain Guy) add support for autoconfiguring Pegasus Mail (not finished) Index: autoconfigure.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/windows/autoconfigure.py,v retrieving revision 1.4 retrieving revision 1.5 diff -C2 -d -r1.4 -r1.5 *** autoconfigure.py 5 Sep 2003 01:15:29 -0000 1.4 --- autoconfigure.py 9 Sep 2003 08:55:51 -0000 1.5 *************** *** 8,11 **** --- 8,12 ---- o Opera Mail (M2) (POP3/SMTP only) o Outlook Express (POP3/SMTP only) + o PocoMail (POP3/SMTP only) To do: *************** *** 83,88 **** except socket.error: portStr = str(port) ! if options["pop3proxy", "listen_ports"].find(portStr) != -1 or \ ! options["smtpproxy", "listen_ports"].find(portStr) != -1: continue else: --- 84,89 ---- except socket.error: portStr = str(port) ! if portStr in options["pop3proxy", "listen_ports"] or \ ! portStr in options["smtpproxy", "listen_ports"]: continue else: *************** *** 412,421 **** # here. ! def configure_outlook_express(key): """Configure OE to use the SpamBayes POP3 and SMTP proxies, and configure SpamBayes to proxy the servers that OE was connecting to.""" # OE stores its configuration in the registry, not a file. ! ! key = key + "\\Software\\Microsoft\\Internet Account Manager\\Accounts" import win32api --- 413,421 ---- # here. ! def configure_outlook_express(): """Configure OE to use the SpamBayes POP3 and SMTP proxies, and configure SpamBayes to proxy the servers that OE was connecting to.""" # OE stores its configuration in the registry, not a file. ! key = "Software\\Microsoft\\Internet Account Manager\\Accounts" import win32api *************** *** 429,433 **** smtp_proxy = smtp_proxy_port ! reg = win32api.RegOpenKeyEx(win32con.HKEY_USERS, key) account_index = 0 while True: --- 429,433 ---- smtp_proxy = smtp_proxy_port ! reg = win32api.RegOpenKeyEx(win32con.HKEY_CURRENT_USER, key) account_index = 0 while True: *************** *** 435,446 **** config = {} try: ! subkey_name = "%s\\%s" % \ ! (key, win32api.RegEnumKey(reg, account_index)) except win32api.error: break account_index += 1 index = 0 ! subkey = win32api.RegOpenKeyEx(win32con.HKEY_USERS, subkey_name, 0, ! win32con.KEY_READ | win32con.KEY_SET_VALUE) while True: # Loop through all the keys --- 435,448 ---- config = {} try: ! subkey_name = "%s\\%s" % (key, ! win32api.RegEnumKey(reg, ! account_index)) except win32api.error: break account_index += 1 index = 0 ! subkey = win32api.RegOpenKeyEx(win32con.HKEY_CURRENT_USER, ! subkey_name, 0, win32con.KEY_READ | ! win32con.KEY_SET_VALUE) while True: # Loop through all the keys *************** *** 657,663 **** if __name__ == "__main__": pmail_ini_dir = "C:\\Program Files\\PMAIL\\MAIL\\ADMIN" - # XXX This is my OE key = "S-1-5-21-95318837-410984162-318601546-13224" - # XXX but I presume it's different for everyone? I'll have to check on - # XXX another machine. #configure_eudora(eudora_ini_dir) #configure_mozilla(mozilla_ini_dir) --- 659,662 ---- *************** *** 665,668 **** #configure_outlook_express() #configure_pocomail() ! configure_pegasus_mail(pmail_ini_dir) pass --- 664,667 ---- #configure_outlook_express() #configure_pocomail() ! #configure_pegasus_mail(pmail_ini_dir) pass From montanaro at users.sourceforge.net Tue Sep 9 12:30:36 2003 From: montanaro at users.sourceforge.net (Skip Montanaro) Date: Tue Sep 9 14:30:46 2003 Subject: [Spambayes-checkins] spambayes README.txt,1.52,1.53 Message-ID: Update of /cvsroot/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv13975 Modified Files: README.txt Log Message: tweak info about training from command line and scoring via procmail Index: README.txt =================================================================== RCS file: /cvsroot/spambayes/spambayes/README.txt,v retrieving revision 1.52 retrieving revision 1.53 diff -C2 -d -r1.52 -r1.53 *** README.txt 9 Sep 2003 07:43:59 -0000 1.52 --- README.txt 9 Sep 2003 18:30:34 -0000 1.53 *************** *** 183,199 **** Once you've trained SpamBayes on your ! collection of know ham and spam, you can use the hammie.py script to classify incoming mail like so: :0 fw:hamlock ! | /usr/local/bin/hammie.py -f -d -p $HOME/.hammie.db ! The above Procmail recipe tells it to run /usr/local/bin/hammie.py in filter ! mode (-f), and to use the training results stored in the dbm-style file ! ~/hammie.db. While hammie.py is runnning, Procmail uses the lock file ! hamlock to prevent multiple invocations from stepping on each others' toes. ! (It's not strictly necessary in this case since no files on-disk are ! modified, but Procmail will still complain if you don't specify a lock ! file.) The result of running hammie.py in filter mode is that Procmail will use the --- 183,199 ---- Once you've trained SpamBayes on your ! collection of know ham and spam, you can use the sb_filter.py script to classify incoming mail like so: :0 fw:hamlock ! | /usr/local/bin/sb_filter.py ! The above Procmail recipe tells it to run /usr/local/bin/sb_filter.py. ! Since no command line arguments are given, it relies on the options file ! specified by the BAYESCUSTOMIZE variable for all parameters. While ! sb_filter.py is runnning, Procmail uses the lock file hamlock to prevent ! multiple invocations from stepping on each others' toes. (It's not strictly ! necessary in this case since no files on-disk are modified, but Procmail ! will still complain if you don't specify a lock file.) The result of running hammie.py in filter mode is that Procmail will use the *************** *** 284,288 **** containing nothing but ham, you can train Spambayes using a command like ! hammie.py -g ~/tmp/newham -s ~/tmp/newspam The above command is command-line-centric (eg. unix, or Windows command --- 284,288 ---- containing nothing but ham, you can train Spambayes using a command like ! sb_mboxtrain.py -g ~/tmp/newham -s ~/tmp/newspam The above command is command-line-centric (eg. unix, or Windows command From anadelonbrin at users.sourceforge.net Tue Sep 9 22:12:23 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Wed Sep 10 00:12:25 2003 Subject: [Spambayes-checkins] spambayes/spambayes hammie.py, 1.9, 1.10 storage.py, 1.29, 1.30 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv31097/spambayes Modified Files: hammie.py storage.py Log Message: A patch for the patched patched patch :) Tidy the whole open storage thing up, and it should once again work for hammie with a pickle. (mmm...ham and pickle...) Index: hammie.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/hammie.py,v retrieving revision 1.9 retrieving revision 1.10 diff -C2 -d -r1.9 -r1.10 *** hammie.py 5 Sep 2003 15:18:03 -0000 1.9 --- hammie.py 10 Sep 2003 04:12:20 -0000 1.10 *************** *** 257,261 **** if needed), 'r' for read-only, 'w' for read-write. """ ! return Hammie(storage.open_storage((filename, mode), useDB)) --- 257,261 ---- if needed), 'r' for read-only, 'w' for read-write. """ ! return Hammie(storage.open_storage(filename, useDB, mode)) Index: storage.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v retrieving revision 1.29 retrieving revision 1.30 diff -C2 -d -r1.29 -r1.30 *** storage.py 9 Sep 2003 06:17:33 -0000 1.29 --- storage.py 10 Sep 2003 04:12:20 -0000 1.30 *************** *** 637,641 **** } ! def open_storage(data_source_name, useDB=True): """Return a storage object appropriate to the given parameters. --- 637,641 ---- } ! def open_storage(data_source_name, useDB=True, mode=None): """Return a storage object appropriate to the given parameters. *************** *** 648,653 **** then a DBDictClassifier is used.""" if useDB: ! if (isinstance(data_source_name, types.StringTypes) and ! data_source_name.find('::') != -1): db_type, rest = data_source_name.split('::', 1) if _storage_types.has_key(db_type.lower()): --- 648,652 ---- then a DBDictClassifier is used.""" if useDB: ! if data_source_name.find('::') != -1: db_type, rest = data_source_name.split('::', 1) if _storage_types.has_key(db_type.lower()): *************** *** 661,666 **** klass = PickledClassifier try: ! if isinstance(data_source_name, type(())): ! return klass(*data_source_name) return klass(data_source_name) except dbmstorage.error, e: --- 660,665 ---- klass = PickledClassifier try: ! if mode is not None: ! return klass(data_source_name, mode) return klass(data_source_name) except dbmstorage.error, e: *************** *** 668,675 **** # We expect this to hit a fair few people, so warn them nicely, # rather than just printing the trackback. ! print >> sys.stderr, """\ ! You do not have a dbm module available to use. You need to either use a ! pickle (see the FAQ), use Python 2.3 (or above), or install a dbm module ! such as bsddb (see http://sf.net/projects/pybsddb).""" import sys sys.exit() --- 667,674 ---- # We expect this to hit a fair few people, so warn them nicely, # rather than just printing the trackback. ! print >> sys.stderr, "You do not have a dbm module available " \ ! "to use. You need to either use a pickle (see the FAQ)" \ ! ", use Python 2.3 (or above), or install a dbm module " \ ! "such as bsddb (see http://sf.net/projects/pybsddb)." import sys sys.exit() From anadelonbrin at users.sourceforge.net Tue Sep 9 22:33:19 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Wed Sep 10 00:33:26 2003 Subject: [Spambayes-checkins] spambayes/contrib nway.py,1.3,1.4 Message-ID: Update of /cvsroot/spambayes/spambayes/contrib In directory sc8-pr-cvs1:/tmp/cvs-serv1738/contrib Modified Files: nway.py Log Message: Update names of scripts in comments/docstrings. Index: nway.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/contrib/nway.py,v retrieving revision 1.3 retrieving revision 1.4 diff -C2 -d -r1.3 -r1.4 *** nway.py 5 Sep 2003 01:15:28 -0000 1.3 --- nway.py 10 Sep 2003 04:33:17 -0000 1.4 *************** *** 24,31 **** series of mboxtrain.py commands: ! mboxtrain.py -f -d python.db -s python -g music -g family -g cars ! mboxtrain.py -f -d music.db -g python -s music -g family -g cars ! mboxtrain.py -f -d family.db -g python -g music -s family -g cars ! mboxtrain.py -f -d cars.db -g python -g music -g family -s cars You'd then compare messages using a %(prog)s command like this: --- 24,31 ---- series of mboxtrain.py commands: ! sb_mboxtrain.py -f -d python.db -s python -g music -g family -g cars ! sb_mboxtrain.py -f -d music.db -g python -s music -g family -g cars ! sb_mboxtrain.py -f -d family.db -g python -g music -s family -g cars ! sb_mboxtrain.py -f -d cars.db -g python -g music -g family -s cars You'd then compare messages using a %(prog)s command like this: From anadelonbrin at users.sourceforge.net Tue Sep 9 22:33:19 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Wed Sep 10 00:33:31 2003 Subject: [Spambayes-checkins] spambayes/scripts sb_client.py, 1.1, 1.2 sb_dbexpimp.py, 1.1, 1.2 sb_filter.py, 1.1, 1.2 sb_notesfilter.py, 1.1, 1.2 sb_server.py, 1.2, 1.3 Message-ID: Update of /cvsroot/spambayes/spambayes/scripts In directory sc8-pr-cvs1:/tmp/cvs-serv1738/scripts Modified Files: sb_client.py sb_dbexpimp.py sb_filter.py sb_notesfilter.py sb_server.py Log Message: Update names of scripts in comments/docstrings. Index: sb_client.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/scripts/sb_client.py,v retrieving revision 1.1 retrieving revision 1.2 diff -C2 -d -r1.1 -r1.2 *** sb_client.py 5 Sep 2003 01:16:45 -0000 1.1 --- sb_client.py 10 Sep 2003 04:33:17 -0000 1.2 *************** *** 1,5 **** #! /usr/bin/env python ! """A client for hammiesrv. Just feed it your mail on stdin, and it spits out the same message --- 1,5 ---- #! /usr/bin/env python ! """A client for sb_xmlrpcserver.py. Just feed it your mail on stdin, and it spits out the same message Index: sb_dbexpimp.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/scripts/sb_dbexpimp.py,v retrieving revision 1.1 retrieving revision 1.2 diff -C2 -d -r1.1 -r1.2 *** sb_dbexpimp.py 5 Sep 2003 01:16:45 -0000 1.1 --- sb_dbexpimp.py 10 Sep 2003 04:33:17 -0000 1.2 *************** *** 1,5 **** #! /usr/bin/env python ! """dbExpImp.py - Bayes database export/import Classes: --- 1,5 ---- #! /usr/bin/env python ! """sb_dbexpimp.py - Bayes database export/import Classes: *************** *** 41,45 **** Usage: ! dbExpImp [options] options: --- 41,45 ---- Usage: ! sb_dbexpimp [options] options: *************** *** 60,81 **** Export pickled mybayes.db into mybayes.db.export as a csv flat file ! dbExpImp -e -d mybayes.db -f mybayes.db.export Import mybayes.eb.export into a new DBM mybayes.db ! dbExpImp -i -D mybayes.db -f mybayes.db.export Export, then import (reorganize) new pickled mybayes.db ! dbExpImp -e -i -n -d mybayes.db -f mybayes.db.export Convert a bayes database from pickle to DBM ! dbExpImp -e -d abayes.db -f abayes.export ! dbExpImp -i -D abayes.db -f abayes.export Create a new database (newbayes.db) from two databases (abayes.db, bbayes.db) ! dbExpImp -e -d abayes.db -f abayes.export ! dbExpImp -e -d bbayes.db -f bbayes.export ! dbExpImp -i -d newbayes.db -f abayes.export ! dbExpImp -i -m -d newbayes.db -f bbayes.export To Do: --- 60,81 ---- Export pickled mybayes.db into mybayes.db.export as a csv flat file ! sb_dbexpimp -e -d mybayes.db -f mybayes.db.export Import mybayes.eb.export into a new DBM mybayes.db ! sb_dbexpimp -i -D mybayes.db -f mybayes.db.export Export, then import (reorganize) new pickled mybayes.db ! sb_dbexpimp -e -i -n -d mybayes.db -f mybayes.db.export Convert a bayes database from pickle to DBM ! sb_dbexpimp -e -d abayes.db -f abayes.export ! sb_dbexpimp -i -D abayes.db -f abayes.export Create a new database (newbayes.db) from two databases (abayes.db, bbayes.db) ! sb_dbexpimp -e -d abayes.db -f abayes.export ! sb_dbexpimp -e -d bbayes.db -f bbayes.export ! sb_dbexpimp -i -d newbayes.db -f abayes.export ! sb_dbexpimp -i -m -d newbayes.db -f bbayes.export To Do: *************** *** 264,266 **** runImport(dbFN, useDBM, newDBM, flatFN) else: ! print >>sys.stderr, __doc__ \ No newline at end of file --- 264,266 ---- runImport(dbFN, useDBM, newDBM, flatFN) else: ! print >>sys.stderr, __doc__ Index: sb_filter.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/scripts/sb_filter.py,v retrieving revision 1.1 retrieving revision 1.2 diff -C2 -d -r1.1 -r1.2 *** sb_filter.py 5 Sep 2003 01:16:45 -0000 1.1 --- sb_filter.py 10 Sep 2003 04:33:17 -0000 1.2 *************** *** 7,11 **** ## ## :0 fw ! ## | hammiefilter.py ## ## Then, you can set up your MUA to pipe ham and spam to it, one at a --- 7,11 ---- ## ## :0 fw ! ## | sb_filter.py ## ## Then, you can set up your MUA to pipe ham and spam to it, one at a Index: sb_notesfilter.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/scripts/sb_notesfilter.py,v retrieving revision 1.1 retrieving revision 1.2 diff -C2 -d -r1.1 -r1.2 *** sb_notesfilter.py 5 Sep 2003 01:16:45 -0000 1.1 --- sb_notesfilter.py 10 Sep 2003 04:33:17 -0000 1.2 *************** *** 1,5 **** #! /usr/bin/env python ! '''notesfilter.py - Lotus Notes Spambayes interface. Classes: --- 1,5 ---- #! /usr/bin/env python ! '''sb_notesfilter.py - Lotus Notes Spambayes interface. Classes: Index: sb_server.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/scripts/sb_server.py,v retrieving revision 1.2 retrieving revision 1.3 diff -C2 -d -r1.2 -r1.3 *** sb_server.py 9 Sep 2003 06:51:00 -0000 1.2 --- sb_server.py 10 Sep 2003 04:33:17 -0000 1.3 *************** *** 1,11 **** #!/usr/bin/env python ! """A POP3 proxy that works with classifier.py, and adds a simple X-Spambayes-Classification header (ham/spam/unsure) to each incoming ! email. You point pop3proxy at your POP3 server, and configure your email client to collect mail from the proxy then filter on the added header. Usage: ! pop3proxy.py [options] [ []] is the name of your real POP3 server is the port number of your real POP3 server, which --- 1,16 ---- #!/usr/bin/env python ! """The primary server for SpamBayes. ! ! Currently serves the web interface, and any configured POP3 and SMTP ! proxies. ! ! The POP3 proxy works with classifier.py, and adds a simple X-Spambayes-Classification header (ham/spam/unsure) to each incoming ! email. You point the proxy at your POP3 server, and configure your email client to collect mail from the proxy then filter on the added header. Usage: ! sb_server.py [options] [ []] is the name of your real POP3 server is the port number of your real POP3 server, which From mhammond at users.sourceforge.net Wed Sep 10 01:33:42 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Wed Sep 10 03:33:46 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000 about.html,1.22,1.23 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000 In directory sc8-pr-cvs1:/tmp/cvs-serv30907 Modified Files: about.html Log Message: Fix absolute references Index: about.html =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/about.html,v retrieving revision 1.22 retrieving revision 1.23 diff -C2 -d -r1.22 -r1.23 *** about.html 1 Sep 2003 07:27:31 -0000 1.22 --- about.html 10 Sep 2003 07:33:39 -0000 1.23 *************** *** 24,28 **** To Spambayes.
        If you want to add the Spam field to your Outlook views, follow these instructions.
        If you need help configuring SpamBayes, see our .
        If you want to add the Spam field to your Outlook views,         follow these instructions.
        If you need help configuring SpamBayes, see our Update of /cvsroot/spambayes/spambayes/Outlook2000/docs In directory sc8-pr-cvs1:/tmp/cvs-serv31745 Modified Files: welcome.html Log Message: Make references relative, and background= for the table to avoid mozilla bug 167262 Index: welcome.html =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/docs/welcome.html,v retrieving revision 1.4 retrieving revision 1.5 diff -C2 -d -r1.4 -r1.5 *** welcome.html 29 Aug 2003 15:18:22 -0000 1.4 --- welcome.html 10 Sep 2003 07:38:51 -0000 1.5 *************** *** 8,15 **** ! ! !
        Logo 

        Welcome to SpamBayes
        --- 8,21 ---- ! ! ! ! ! ! !
        Logo  

        Welcome to SpamBayes
        *************** *** 107,112 ****

        Outlook does not allow us to automatically add the spam score to your Outlook folder ! views - but you can do it manually by following these instructions.

        --- 113,117 ----

        Outlook does not allow us to automatically add the spam score to your Outlook folder ! views - but you can do it manually by following these instructions.

        From mhammond at users.sourceforge.net Wed Sep 10 01:42:49 2003 From: mhammond at users.sourceforge.net (Mark Hammond) Date: Wed Sep 10 03:42:52 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000 manager.py,1.86,1.87 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000 In directory sc8-pr-cvs1:/tmp/cvs-serv32476 Modified Files: manager.py Log Message: Call new new close() method on the classifier rather than hacking our own. Index: manager.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/manager.py,v retrieving revision 1.86 retrieving revision 1.87 diff -C2 -d -r1.86 -r1.87 *** manager.py 7 Sep 2003 23:42:19 -0000 1.86 --- manager.py 10 Sep 2003 07:42:45 -0000 1.87 *************** *** 172,175 **** --- 172,177 ---- def open_bayes(self): raise NotImplementedError + def close_bayes(self, bayes): + bayes.close() class PickleStorageManager(BasicStorageManager): *************** *** 177,182 **** def open_bayes(self): return bayes_storage.PickledClassifier(self.bayes_filename) - def close_bayes(self, bayes): - pass def open_mdb(self): return cPickle.load(open(self.mdb_filename, 'rb')) --- 179,182 ---- *************** *** 196,202 **** fname = self.bayes_filename.encode(filesystem_encoding) return bayes_storage.DBDictClassifier(fname) - def close_bayes(self, bayes): - bayes.db.close() - bayes.dbm.close() def open_mdb(self): fname = self.mdb_filename.encode(filesystem_encoding) --- 196,199 ---- From anadelonbrin at users.sourceforge.net Wed Sep 10 02:24:26 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Wed Sep 10 04:24:36 2003 Subject: [Spambayes-checkins] website faq.txt,1.40,1.41 Message-ID: Update of /cvsroot/spambayes/website In directory sc8-pr-cvs1:/tmp/cvs-serv7485 Modified Files: faq.txt Log Message: Update the FAQ to help users of v8 of the plug-in to find the data folder. Index: faq.txt =================================================================== RCS file: /cvsroot/spambayes/website/faq.txt,v retrieving revision 1.40 retrieving revision 1.41 diff -C2 -d -r1.40 -r1.41 *** faq.txt 5 Sep 2003 09:21:02 -0000 1.40 --- faq.txt 10 Sep 2003 08:24:24 -0000 1.41 *************** *** 489,502 **** back in to recover from a corrupted database, or for any other reason. ! This directory is located in the "Application Data" directory. On Windows ! NT/2000/XP, this will probably be: C:\\Documents and Settings\\[username]\\Application Data\\Spambayes ! On other versions of Windows, this will probably be: C:\\Windows\\Application Data\\Spambayes ! Note that this folder may be hidden. --- 489,505 ---- back in to recover from a corrupted database, or for any other reason. ! This directory is located in the "Application Data" directory. If you have ! version 008 of the plug-in, or higher, you can locate this directory by ! using the `Show Data Folder` button on the `Advanced` tab of the main ! `SpamBayes` manager dialog. If you need to locate it by hand, on Windows ! NT/2000/XP, it will probably be: C:\\Documents and Settings\\[username]\\Application Data\\Spambayes ! On other versions of Windows, it will probably be: C:\\Windows\\Application Data\\Spambayes ! Note that the 'Application Data' folder may be hidden. From montanaro at users.sourceforge.net Wed Sep 10 08:25:22 2003 From: montanaro at users.sourceforge.net (Skip Montanaro) Date: Wed Sep 10 10:25:26 2003 Subject: [Spambayes-checkins] spambayes/spambayes storage.py,1.30,1.31 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv6711 Modified Files: storage.py Log Message: indicate whether or not the classifier's __init__ function has a mode arg and call appropriately Index: storage.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v retrieving revision 1.30 retrieving revision 1.31 diff -C2 -d -r1.30 -r1.31 *** storage.py 10 Sep 2003 04:12:20 -0000 1.30 --- storage.py 10 Sep 2003 14:25:19 -0000 1.31 *************** *** 631,638 **** return repr(self.invalid_name) ! _storage_types = {"dbm" : DBDictClassifier, ! "pickle" : PickledClassifier, ! "pgsql" : PGClassifier, ! "mysql" : mySQLClassifier, } --- 631,640 ---- return repr(self.invalid_name) ! # values are classifier class and True if it accepts a mode ! # arg, False otherwise ! _storage_types = {"dbm" : (DBDictClassifier, True), ! "pickle" : (PickledClassifier, False), ! "pgsql" : (PGClassifier, False), ! "mysql" : (mySQLClassifier, False), } *************** *** 651,666 **** db_type, rest = data_source_name.split('::', 1) if _storage_types.has_key(db_type.lower()): ! klass = _storage_types[db_type.lower()] data_source_name = rest else: raise NoSuchClassifierError(db_type) else: ! klass = DBDictClassifier else: ! klass = PickledClassifier try: ! if mode is not None: return klass(data_source_name, mode) ! return klass(data_source_name) except dbmstorage.error, e: if str(e) == "No dbm modules available!": --- 653,669 ---- db_type, rest = data_source_name.split('::', 1) if _storage_types.has_key(db_type.lower()): ! klass, supports_mode = _storage_types[db_type.lower()] data_source_name = rest else: raise NoSuchClassifierError(db_type) else: ! klass, supports_mode = _storage_types["dbm"] else: ! klass, supports_mode = _storage_types["pickle"] try: ! if supports_mode and mode is not None: return klass(data_source_name, mode) ! else: ! return klass(data_source_name) except dbmstorage.error, e: if str(e) == "No dbm modules available!": From richiehindle at users.sourceforge.net Wed Sep 10 15:08:26 2003 From: richiehindle at users.sourceforge.net (Richie Hindle) Date: Wed Sep 10 17:08:30 2003 Subject: [Spambayes-checkins] spambayes/spambayes storage.py,1.31,1.32 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv2019 Modified Files: storage.py Log Message: Bug 803501: Fix the "No dbm modules available" message to print rather than crash. Here's a thought: if people would run their edits before committing them, we'd have fewer bug reports. Just an idea. Also prepended a newline to the error message, so that it doesn't run into the end of the "Loading database..." message when running sb_server.py. Index: storage.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v retrieving revision 1.31 retrieving revision 1.32 diff -C2 -d -r1.31 -r1.32 *** storage.py 10 Sep 2003 14:25:19 -0000 1.31 --- storage.py 10 Sep 2003 21:08:24 -0000 1.32 *************** *** 670,678 **** # We expect this to hit a fair few people, so warn them nicely, # rather than just printing the trackback. ! print >> sys.stderr, "You do not have a dbm module available " \ "to use. You need to either use a pickle (see the FAQ)" \ ", use Python 2.3 (or above), or install a dbm module " \ "such as bsddb (see http://sf.net/projects/pybsddb)." - import sys sys.exit() --- 670,677 ---- # We expect this to hit a fair few people, so warn them nicely, # rather than just printing the trackback. ! print >> sys.stderr, "\nYou do not have a dbm module available " \ "to use. You need to either use a pickle (see the FAQ)" \ ", use Python 2.3 (or above), or install a dbm module " \ "such as bsddb (see http://sf.net/projects/pybsddb)." sys.exit() From anthonybaxter at users.sourceforge.net Wed Sep 10 22:57:57 2003 From: anthonybaxter at users.sourceforge.net (Anthony Baxter) Date: Thu Sep 11 00:58:00 2003 Subject: [Spambayes-checkins] website faq.txt,1.41,1.42 Message-ID: Update of /cvsroot/spambayes/website In directory sc8-pr-cvs1:/tmp/cvs-serv23622 Modified Files: faq.txt Log Message: add a couple of notes about whitelisting: 1. Use your mailer to intercept the emails before spambayes or 2. Send code. Index: faq.txt =================================================================== RCS file: /cvsroot/spambayes/website/faq.txt,v retrieving revision 1.41 retrieving revision 1.42 diff -C2 -d -r1.41 -r1.42 *** faq.txt 10 Sep 2003 08:24:24 -0000 1.41 --- faq.txt 11 Sep 2003 04:57:55 -0000 1.42 *************** *** 1087,1090 **** --- 1087,1100 ---- tools like SpamBayes fill. + If you really need whitelisting, consider implementing rules in your + mailer to intercept the messages before they're passed to spambayes. + + None of the existing Spambayes developers have an interest in + implementing whitelisting - if it's something you feel really strongly + about, download the source and look into making a patch. If someone + implements whitelisting, including all the user interface side needed + to manage it, there's no reason it wouldn't be accepted into the source + code. + .. _related page: related.html From anthonybaxter at users.sourceforge.net Thu Sep 11 00:57:44 2003 From: anthonybaxter at users.sourceforge.net (Anthony Baxter) Date: Thu Sep 11 02:57:48 2003 Subject: [Spambayes-checkins] spambayes NEWTRICKS.txt,NONE,1.1 Message-ID: Update of /cvsroot/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv10913 Added Files: NEWTRICKS.txt Log Message: a file to record ideas that have and haven't been tried. --- NEW FILE: NEWTRICKS.txt --- This file is for ideas that have or have not yet been tried, and the results of these ideas. - If the email parser was able to emit warnings about misformed MIME that it encountered, we could turn this warnings into tokens. - The numeric entity support looks like it will only support 8-bit characters - an obvious problem is from wacky wide-char entities. From anthonybaxter at users.sourceforge.net Thu Sep 11 01:04:25 2003 From: anthonybaxter at users.sourceforge.net (Anthony Baxter) Date: Thu Sep 11 03:04:28 2003 Subject: [Spambayes-checkins] website background.ht,1.16,1.17 Message-ID: Update of /cvsroot/spambayes/website In directory sc8-pr-cvs1:/tmp/cvs-serv12193 Modified Files: background.ht Log Message: ascribe blame where blame is due. Thanks to Gary (and Tim) for updating my view of history. Index: background.ht =================================================================== RCS file: /cvsroot/spambayes/website/background.ht,v retrieving revision 1.16 retrieving revision 1.17 diff -C2 -d -r1.16 -r1.17 *** background.ht 4 Aug 2003 02:49:59 -0000 1.16 --- background.ht 11 Sep 2003 07:04:22 -0000 1.17 *************** *** 136,144 **** with an 'unsure' answer.

        !

        ! These two central limit ! approaches were dropped after Tim, Gary and Rob Hooft produced a combining ! scheme using chi-squared probabilities. This is now the default combining ! scheme.

        Chi-combining is similar to the central limit approaches, but it doesn't have the annoying training problems that central limit approaches suffered --- 136,149 ---- with an 'unsure' answer.

        !

        The central limit approaches were dropped after Gary suggested a ! novel but well-founded combining scheme using ! chi-squared ! probabilities. ! Tim implemented it, including a unique routine to compute ! -2*sum(ln p_i) without logarithms and without underflow worries. ! Many, many testers reported excellent results. This is now the ! default combining scheme. Some end cases were improved when ! Rob Hooft discovered a cleaner way to combine the internal overall ham ! and spam scores, which improved detection of the "middle ground".

        Chi-combining is similar to the central limit approaches, but it doesn't have the annoying training problems that central limit approaches suffered From anthonybaxter at users.sourceforge.net Thu Sep 11 01:33:51 2003 From: anthonybaxter at users.sourceforge.net (Anthony Baxter) Date: Thu Sep 11 03:33:58 2003 Subject: [Spambayes-checkins] website faq.txt,1.42,1.43 Message-ID: Update of /cvsroot/spambayes/website In directory sc8-pr-cvs1:/tmp/cvs-serv17601 Modified Files: faq.txt Log Message: consistent c12n of SpamBayes (that's what it's called on the front page of the website) Index: faq.txt =================================================================== RCS file: /cvsroot/spambayes/website/faq.txt,v retrieving revision 1.42 retrieving revision 1.43 diff -C2 -d -r1.42 -r1.43 *** faq.txt 11 Sep 2003 04:57:55 -0000 1.42 --- faq.txt 11 Sep 2003 07:33:48 -0000 1.43 *************** *** 1,4 **** ==================================== ! Spambayes Frequently Asked Questions ==================================== --- 1,4 ---- ==================================== ! SpamBayes Frequently Asked Questions ==================================== *************** *** 17,24 **** This is a work in progress. Please feel free to ask questions and/or ! provide answers; For help with Spambayes, send email to the `Spambayes mailing list`_. ! .. _Spambayes mailing list: mailto:spambayes@python.org --- 17,24 ---- This is a work in progress. Please feel free to ask questions and/or ! provide answers; For help with SpamBayes, send email to the `SpamBayes mailing list`_. ! .. _SpamBayes mailing list: mailto:spambayes@python.org *************** *** 26,39 **** ======== ! What is Spambayes? ------------------ ! Spambayes is a tool used to segregate unwanted mail (spam) from the mail you ! want (ham). Before Spambayes can be your spam filter of choice you need to train it on representative samples of email you receive. After it's been ! trained, you use Spambayes to classify new mail according to its spamminess and hamminess qualities. ! To train Spambayes (which you don't need to do if you're going to be using the POP3 proxy to classify messages, but you'll get better results from the outset if you do) you need to save your incoming email for awhile, --- 26,39 ---- ======== ! What is SpamBayes? ------------------ ! SpamBayes is a tool used to segregate unwanted mail (spam) from the mail you ! want (ham). Before SpamBayes can be your spam filter of choice you need to train it on representative samples of email you receive. After it's been ! trained, you use SpamBayes to classify new mail according to its spamminess and hamminess qualities. ! To train SpamBayes (which you don't need to do if you're going to be using the POP3 proxy to classify messages, but you'll get better results from the outset if you do) you need to save your incoming email for awhile, *************** *** 42,46 **** and the nature of what spam looks like change over time. Once you've collected a fair portion of each (anything is better than nothing, but it ! helps to have a couple hundred of each), you can tell Spambayes, "Here's my ham and my spam". It will then process that mail and save information about different patterns which appear in ham and spam. That information is then --- 42,46 ---- and the nature of what spam looks like change over time. Once you've collected a fair portion of each (anything is better than nothing, but it ! helps to have a couple hundred of each), you can tell SpamBayes, "Here's my ham and my spam". It will then process that mail and save information about different patterns which appear in ham and spam. That information is then *************** *** 48,52 **** below for details. ! When Spambayes filters your email, it compares each unclassified message against the information it saved from training and makes a decision about whether it thinks the message qualifies as ham or spam, or if it's unsure --- 48,52 ---- below for details. ! When SpamBayes filters your email, it compares each unclassified message against the information it saved from training and makes a decision about whether it thinks the message qualifies as ham or spam, or if it's unsure *************** *** 54,58 **** message, either by adding a header (X-Spambayes-Classification: spam|ham|unsure), modifying the To: or Subject: headers, or adding a "Spam" ! field to the message. Depending on which Spambayes application you are using, it may then filter this message for you, or you can set up your own filters (to file away suspected spam into its own mail folder, for example). --- 54,58 ---- message, either by adding a header (X-Spambayes-Classification: spam|ham|unsure), modifying the To: or Subject: headers, or adding a "Spam" ! field to the message. Depending on which SpamBayes application you are using, it may then filter this message for you, or you can set up your own filters (to file away suspected spam into its own mail folder, for example). *************** *** 87,103 **** ------------------------------------ ! There are four mailing lists which support Spambayes: ! 1. The `Spambayes list`_ provides a place for users to get help with ! Spambayes or help other users. ! 2. The `Spambayes developers list`_ provides a forum for people maintaining and improving the package. ! 3. The `Spambayes announcements list`_ is a low-volume list where announcements about new releases are posted. ! 4. The `Spambayes checkins list`_ receives summaries of all the ! changes to the Spambayes software. This is generally only of interest to developers. --- 87,103 ---- ------------------------------------ ! There are four mailing lists which support SpamBayes: ! 1. The `SpamBayes list`_ provides a place for users to get help with ! SpamBayes or help other users. ! 2. The `SpamBayes developers list`_ provides a forum for people maintaining and improving the package. ! 3. The `SpamBayes announcements list`_ is a low-volume list where announcements about new releases are posted. ! 4. The `SpamBayes checkins list`_ receives summaries of all the ! changes to the SpamBayes software. This is generally only of interest to developers. *************** *** 113,123 **** messages. ! .. _Spambayes list: http://mail.python.org/mailman/listinfo/spambayes ! .. _Spambayes developers list: http://mail.python.org/mailman/listinfo/spambayes-dev ! .. _Spambayes announcements list: http://mail.python.org/mailman/listinfo/spambayes-announce ! .. _Spambayes checkins list: http://mail.python.org/mailman/listinfo/spambayes-checkins ! What do I need to install Spambayes? ------------------------------------ --- 113,123 ---- messages. ! .. _SpamBayes list: http://mail.python.org/mailman/listinfo/spambayes ! .. _SpamBayes developers list: http://mail.python.org/mailman/listinfo/spambayes-dev ! .. _SpamBayes announcements list: http://mail.python.org/mailman/listinfo/spambayes-announce ! .. _SpamBayes checkins list: http://mail.python.org/mailman/listinfo/spambayes-checkins ! What do I need to install SpamBayes? ------------------------------------ *************** *** 139,146 **** ! Is there a high level summary that shows how Spambayes works? ------------------------------------------------------------- ! There are eight main components to the Spambayes system: 1. A database. Loosely speaking, this is a collection of words and --- 139,146 ---- ! Is there a high level summary that shows how SpamBayes works? ------------------------------------------------------------- ! There are eight main components to the SpamBayes system: 1. A database. Loosely speaking, this is a collection of words and *************** *** 175,179 **** +-----------------+ +------------+ +-------------+ ! | Outlook Express | | Spambayes | | | | (or similar) | <--> | POP3 proxy | <--> | POP3 server | | | | | | | --- 175,179 ---- +-----------------+ +------------+ +-------------+ ! | Outlook Express | | SpamBayes | | | | (or similar) | <--> | POP3 proxy | <--> | POP3 server | | | | | | | *************** *** 205,209 **** +-----------------+ +------------+ +-------------+ ! | Outlook Express | | Spambayes | | | | (or similar) | <--> | SMTP proxy | <--> | SMTP server | | | | | | | --- 205,209 ---- +-----------------+ +------------+ +-------------+ ! | Outlook Express | | SpamBayes | | | | (or similar) | <--> | SMTP proxy | <--> | SMTP server | | | | | | | *************** *** 254,258 **** as ham and folders that contain mail to train as spam, and the filter does this for you. You can also designate folders to filter, along with ! a folder for messages Spambayes is unsure about, and a folder for suspected spam. When new mail arrives, the filter will move mail to the appropriate location (ham is left in the original folder). --- 254,258 ---- as ham and folders that contain mail to train as spam, and the filter does this for you. You can also designate folders to filter, along with ! a folder for messages SpamBayes is unsure about, and a folder for suspected spam. When new mail arrives, the filter will move mail to the appropriate location (ham is left in the original folder). *************** *** 269,273 **** As well as these components, there's also a whole pile of utility scripts, test harnesses and so on - see README.txt and TESTING.txt in the ! spambayes distribution for more information. --- 269,273 ---- As well as these components, there's also a whole pile of utility scripts, test harnesses and so on - see README.txt and TESTING.txt in the ! SpamBayes distribution for more information. *************** *** 276,280 **** ! Does Spambayes work with Outlook Express? ----------------------------------------- --- 276,280 ---- ! Does SpamBayes work with Outlook Express? ----------------------------------------- *************** *** 297,306 **** ! What clients will Spambayes work with in general? ------------------------------------------------- ! Spambayes will work with most POP3 or IMAP compatible clients. How you implement depends on your local architecture. Users with access to procmail ! can just write a recipe that invokes spambayes like this:: :0fw --- 297,306 ---- ! What clients will SpamBayes work with in general? ------------------------------------------------- ! SpamBayes will work with most POP3 or IMAP compatible clients. How you implement depends on your local architecture. Users with access to procmail ! can just write a recipe that invokes SpamBayes like this:: :0fw *************** *** 345,349 **** Users limited to POP3/IMAP communications to the server can use the POP3_ or ! IMAP_ proxies which are part of the Spambayes source. .. _POP3: http://spambayes.sf.net/applications.html#pop3 --- 345,349 ---- Users limited to POP3/IMAP communications to the server can use the POP3_ or ! IMAP_ proxies which are part of the SpamBayes source. .. _POP3: http://spambayes.sf.net/applications.html#pop3 *************** *** 351,355 **** ! How do I configure Eudora for use with Spambayes? ------------------------------------------------- --- 351,355 ---- ! How do I configure Eudora for use with SpamBayes? ------------------------------------------------- *************** *** 436,440 **** ! Do I have to have Python installed to use Spambayes with Outlook? ----------------------------------------------------------------- --- 436,440 ---- ! Do I have to have Python installed to use SpamBayes with Outlook? ----------------------------------------------------------------- *************** *** 443,447 **** ! Will Spambayes work with Outlook 2000 connecting to an Exchange 2000 server? ---------------------------------------------------------------------------- --- 443,447 ---- ! Will SpamBayes work with Outlook 2000 connecting to an Exchange 2000 server? ---------------------------------------------------------------------------- *************** *** 454,458 **** alpha3 source releases and CVS). Details about how to enable this feature can be found in the "configuration.html" file in the "docs" directory in ! the folder Spambayes was installed to. You can also set up mail that is trained using the "Delete as spam" button to be marked as read, rather than all mail classified as spam. However, you should also see the --- 454,458 ---- alpha3 source releases and CVS). Details about how to enable this feature can be found in the "configuration.html" file in the "docs" directory in ! the folder SpamBayes was installed to. You can also set up mail that is trained using the "Delete as spam" button to be marked as read, rather than all mail classified as spam. However, you should also see the *************** *** 580,588 **** register a hit on the spammer's server. ! Using Spambayes =============== ! How do I train Spambayes (web method)? -------------------------------------- --- 580,588 ---- register a hit on the spammer's server. ! Using SpamBayes =============== ! How do I train SpamBayes (web method)? -------------------------------------- *************** *** 606,610 **** ! How do I train Spambayes (forward/bounce method)? ------------------------------------------------- --- 606,610 ---- ! How do I train SpamBayes (forward/bounce method)? ------------------------------------------------- *************** *** 626,635 **** ! How do I train Spambayes (command line method)? ----------------------------------------------- Given a pair of Unix mailbox format files (each message starts with a line which begins with 'From '), one containing nothing but spam and the other ! containing nothing but ham, you can train Spambayes using a command like:: hammie.py -g ~/tmp/newham -s ~/tmp/newspam --- 626,635 ---- ! How do I train SpamBayes (command line method)? ----------------------------------------------- Given a pair of Unix mailbox format files (each message starts with a line which begins with 'From '), one containing nothing but spam and the other ! containing nothing but ham, you can train SpamBayes using a command like:: hammie.py -g ~/tmp/newham -s ~/tmp/newspam *************** *** 639,643 **** ! How do I train Spambayes (Outlook plugin)? ----------------------------------------------- --- 639,643 ---- ! How do I train SpamBayes (Outlook plugin)? ----------------------------------------------- *************** *** 656,660 **** Once a message has been [correctly] trained there is no need to keep it ! around. However, Spambayes' accuracy is dependent upon having a "sufficient" sample from which to make its decisions. Therefore, most users retain a fair amount of spam in the event that they may wish to rebuild the corpus from --- 656,660 ---- Once a message has been [correctly] trained there is no need to keep it ! around. However, SpamBayes' accuracy is dependent upon having a "sufficient" sample from which to make its decisions. Therefore, most users retain a fair amount of spam in the event that they may wish to rebuild the corpus from *************** *** 662,666 **** Of course, this begs the question: "how much is enough?" That is where the ! "art" of spambayes meets the science. Some users keep as many as several thousand [recent] spam (as well as a similar number of ham). That is not to say that you won't have excellent results with a tenth (or less) of that --- 662,666 ---- Of course, this begs the question: "how much is enough?" That is where the ! "art" of SpamBayes meets the science. Some users keep as many as several thousand [recent] spam (as well as a similar number of ham). That is not to say that you won't have excellent results with a tenth (or less) of that *************** *** 669,673 **** ! Why did Spambayes mark this obvious spam "unsure"? -------------------------------------------------- --- 669,673 ---- ! Why did SpamBayes mark this obvious spam "unsure"? -------------------------------------------------- *************** *** 689,696 **** ! Spambayes doesn't seem to catch much spam. What gives? ------------------------------------------------------- ! Initially, Spambayes will not be able to distinguish spams from hams. With no training inputs, the classifier will simply mark everything unsure. Once you start training the classifier on a representative set of spams and hams --- 689,696 ---- ! SpamBayes doesn't seem to catch much spam. What gives? ------------------------------------------------------- ! Initially, SpamBayes will not be able to distinguish spams from hams. With no training inputs, the classifier will simply mark everything unsure. Once you start training the classifier on a representative set of spams and hams *************** *** 712,716 **** should move incorrectly classified messages to their correct locations and retrain (which may happen automatically, depending on how you're ! using Spambayes). --- 712,716 ---- should move incorrectly classified messages to their correct locations and retrain (which may happen automatically, depending on how you're ! using SpamBayes). *************** *** 728,732 **** ! How do I configure Spambayes? ----------------------------- --- 728,732 ---- ! How do I configure SpamBayes? ----------------------------- *************** *** 805,809 **** SpamBayes looks for your configuration file in three places - if it can't find it, then, obviously, your options will not be loaded. The first place ! that Spambayes checks is the environment variable BAYESCUSTOMIZE. You can set this to the path of your configuration file, wherever it is, and it will be loaded. You can also specify more than one file, separated by the --- 805,809 ---- SpamBayes looks for your configuration file in three places - if it can't find it, then, obviously, your options will not be loaded. The first place ! that SpamBayes checks is the environment variable BAYESCUSTOMIZE. You can set this to the path of your configuration file, wherever it is, and it will be loaded. You can also specify more than one file, separated by the *************** *** 858,862 **** ------------------------------------- ! While Spambayes does an excellent job of classifying incoming mail, it is only as good as the data on which it was trained. Here are some tips to help you create a good training set: --- 858,862 ---- ------------------------------------- ! While SpamBayes does an excellent job of classifying incoming mail, it is only as good as the data on which it was trained. Here are some tips to help you create a good training set: *************** *** 864,870 **** * Don't use old mail. The characteristics of your email change over time, sometimes subtly, sometimes dramatically, so it's best to use very recent ! mail to train Spambayes. If you've abandoned an email address in the past because it was getting spammed heavily, there are probably some clues in ! mail sent to your old address which would bias Spambayes. * Check and recheck your training collections. While you are manually --- 864,870 ---- * Don't use old mail. The characteristics of your email change over time, sometimes subtly, sometimes dramatically, so it's best to use very recent ! mail to train SpamBayes. If you've abandoned an email address in the past because it was getting spammed heavily, there are probably some clues in ! mail sent to your old address which would bias SpamBayes. * Check and recheck your training collections. While you are manually *************** *** 874,878 **** ! Can Spambayes be used to perform n-way classification? ------------------------------------------------------ --- 874,878 ---- ! Can SpamBayes be used to perform n-way classification? ------------------------------------------------------ *************** *** 881,885 **** `_. A demonstration script which performs n-way classification was also recently added to the ``contrib`` directory of ! the Spambayes CVS repository. --- 881,885 ---- `_. A demonstration script which performs n-way classification was also recently added to the ``contrib`` directory of ! the SpamBayes CVS repository. *************** *** 1010,1014 **** ! Are there plans to develop a server-side spambayes solution? ------------------------------------------------------------ --- 1010,1014 ---- ! Are there plans to develop a server-side SpamBayes solution? ------------------------------------------------------------ *************** *** 1018,1022 **** on what is spam and what is not, then this might work for you, but otherwise you really have to have individual databases for each user. Either way, you ! should be able to modify spambayes easily enough to fit into your setup. Some people have in fact done this and have been kind enough to donate `notes about how they have gone about it`_. If you also do this but in some other --- 1018,1022 ---- on what is spam and what is not, then this might work for you, but otherwise you really have to have individual databases for each user. Either way, you ! should be able to modify SpamBayes easily enough to fit into your setup. Some people have in fact done this and have been kind enough to donate `notes about how they have gone about it`_. If you also do this but in some other *************** *** 1088,1094 **** If you really need whitelisting, consider implementing rules in your ! mailer to intercept the messages before they're passed to spambayes. ! None of the existing Spambayes developers have an interest in implementing whitelisting - if it's something you feel really strongly about, download the source and look into making a patch. If someone --- 1088,1094 ---- If you really need whitelisting, consider implementing rules in your ! mailer to intercept the messages before they're passed to SpamBayes. ! None of the existing SpamBayes developers have an interest in implementing whitelisting - if it's something you feel really strongly about, download the source and look into making a patch. If someone *************** *** 1103,1112 **** --------------------------------------- ! If you're not a Spambayes developer simply send your corrections or proposed ! questions and answers to the `Spambayes developers mailing list`_. If you are a developer you need a recent version of Docutils_ and the tools/html.py script from that distribution must be in a directory on your PATH. ! .. _Spambayes developers mailing list: spambayes-dev@python.org .. _Docutils: http://docutils.sf.net/ --- 1103,1112 ---- --------------------------------------- ! If you're not a SpamBayes developer simply send your corrections or proposed ! questions and answers to the `SpamBayes developers mailing list`_. If you are a developer you need a recent version of Docutils_ and the tools/html.py script from that distribution must be in a directory on your PATH. ! .. _SpamBayes developers mailing list: spambayes-dev@python.org .. _Docutils: http://docutils.sf.net/ From ta-meyer at ihug.co.nz Thu Sep 11 20:45:39 2003 From: ta-meyer at ihug.co.nz (Tony Meyer) Date: Thu Sep 11 03:45:45 2003 Subject: [Spambayes-checkins] spambayes/spambayes storage.py,1.31,1.32 In-Reply-To: <1ED4ECF91CDED24C8D012BCF2B034F1303346327@its-xchg4.massey.ac.nz> Message-ID: <1ED4ECF91CDED24C8D012BCF2B034F13026F2927@its-xchg4.massey.ac.nz> > Here's a thought: if people would > run their edits before committing them, we'd have fewer bug > reports. Just an idea. To be fair, I tested that the error message printed out correctly when I added it, and Skip probably checked that messages were going to stderr when he added those, just not the specific case of having no dbm modules available. It's a big ask to check that *every* part of the code still works. This is really a case for a more comprehensive suite of tests, so that we *can* check every part of the code still works. I've checked in a unittest (in test_storage.py) that tests that we still fail (correctly) when no dbm modules are available. This is my first ever unittest, so I'd welcome any critiques. =Tony Meyer From anadelonbrin at users.sourceforge.net Thu Sep 11 01:47:15 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Thu Sep 11 03:47:18 2003 Subject: [Spambayes-checkins] spambayes/spambayes/test test_storage.py, 1.2, 1.3 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes/test In directory sc8-pr-cvs1:/tmp/cvs-serv20097/spambayes/test Modified Files: test_storage.py Log Message: Add a unittest to check that we correctly fail when no dbm modules are available. Index: test_storage.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/test/test_storage.py,v retrieving revision 1.2 retrieving revision 1.3 diff -C2 -d -r1.2 -r1.3 *** test_storage.py 25 Jul 2003 05:17:22 -0000 1.2 --- test_storage.py 11 Sep 2003 07:47:13 -0000 1.3 *************** *** 142,145 **** --- 142,167 ---- _StorageTestBase.tearDown(self) + def fail_open_best(self, *args): + from spambayes import dbmstorage + raise dbmstorage.error("No dbm modules available!") + def success(self, *args): + self.succeeded = True + def testNoDBMAvailable(self): + import tempfile + from spambayes.storage import open_storage + DBDictClassifier_load = DBDictClassifier.load + DBDictClassifier.load = self.fail_open_best + sys_exit = sys.exit + sys.exit = self.success + self.succeeded = False + db_name = tempfile.mktemp("nodbmtest") + s = open_storage(db_name, True) + DBDictClassifier.load = DBDictClassifier_load + sys.exit = sys_exit + if not self.succeeded: + self.fail() + if os.path.isfile(db_name): + os.remove(db_name) + def suite(): # We dont want our base class run From anadelonbrin at users.sourceforge.net Thu Sep 11 01:47:55 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Thu Sep 11 03:47:58 2003 Subject: [Spambayes-checkins] spambayes setup.py,1.19,1.20 Message-ID: Update of /cvsroot/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv20208 Modified Files: setup.py Log Message: If the old files exist, offer to delete them. Based on the patches worked up on spambayes-dev by Skip, Richie and me. Index: setup.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/setup.py,v retrieving revision 1.19 retrieving revision 1.20 diff -C2 -d -r1.19 -r1.20 *** setup.py 7 Jul 2003 00:09:27 -0000 1.19 --- setup.py 11 Sep 2003 07:47:52 -0000 1.20 *************** *** 1,4 **** --- 1,6 ---- #!/usr/bin/env python + import os + import sys if sys.version < '2.2': *************** *** 25,64 **** from spambayes import __version__ ! setup( ! name='spambayes', ! version = __version__, ! description = "Spam classification system", ! author = "the spambayes project", ! author_email = "spambayes@python.org", ! url = "http://spambayes.sourceforge.net", ! scripts=['unheader.py', ! 'hammie.py', ! 'hammiecli.py', ! 'hammiesrv.py', ! 'hammiefilter.py', ! 'pop3proxy.py', ! 'smtpproxy.py', ! 'proxytee.py', ! 'dbExpImp.py', ! 'mboxtrain.py', ! 'imapfilter.py', ! 'notesfilter.py', ! ], ! packages = [ ! 'spambayes', ! 'spambayes.resources', ! ], ! classifiers = [ ! 'Development Status :: 4 - Beta', ! 'Environment :: Console', ! 'License :: OSI Approved :: Python Software Foundation License', ! 'Operating System :: POSIX', ! 'Operating System :: MacOS :: MacOS X', ! 'Operating System :: Microsoft :: Windows :: Windows 95/98/2000', ! 'Operating System :: Microsoft :: Windows :: Windows NT/2000', ! 'Programming Language :: Python', ! 'Intended Audience :: End Users/Desktop', ! 'Topic :: Communications :: Email :: Filters', ! 'Topic :: Communications :: Email :: Post-Office :: POP3', ] ! ) --- 27,106 ---- from spambayes import __version__ ! import distutils.command.install_scripts ! parent = distutils.command.install_scripts.install_scripts ! class install_scripts(parent): ! old_scripts=[ ! 'unheader.py', ! 'hammie.py', ! 'hammiecli.py', ! 'hammiesrv.py', ! 'hammiefilter.py', ! 'pop3proxy.py', ! 'smtpproxy.py', ! 'proxytee.py', ! 'dbExpImp.py', ! 'mboxtrain.py', ! 'imapfilter.py', ! 'notesfilter.py', ] ! ! def run(self): ! err = False ! for s in self.old_scripts: ! s = os.path.join(self.install_dir, s) ! if os.path.exists(s): ! print >> sys.stderr, "Error: old script", s, "still exists." ! err = True ! if err: ! print >>sys.stderr, "Do you want to delete these scripts? (y/n)" ! answer = raw_input("") ! if answer == "y": ! for s in self.old_scripts: ! s = os.path.join(self.install_dir, s) ! try: ! os.remove(s) ! print "Removed", s ! except OSError: ! pass ! return parent.run(self) ! ! setup( ! name='spambayes', ! version = __version__, ! description = "Spam classification system", ! author = "the spambayes project", ! author_email = "spambayes@python.org", ! url = "http://spambayes.sourceforge.net", ! cmdclass = {'build_py': install_scripts}, ! scripts=['scripts/sb_client.py', ! 'scripts/sb_dbexpimp.py', ! 'scripts/sb_filter.py', ! 'scripts/sb_imapfilter.py', ! 'scripts/sb_mailsort.py', ! 'scripts/sb_mboxtrain.py', ! 'scripts/sb_notesfilter.py', ! 'scripts/sb_pop3dnd.py', ! 'scripts/sb_server.py', ! 'scripts/sb_smtpproxy.py', ! 'scripts/sb_unheader.py', ! 'scripts/sb_upload.py', ! 'scripts/sb_xmlrpcserver.py', ! ], ! packages = [ ! 'spambayes', ! 'spambayes.resources', ! ], ! classifiers = [ ! 'Development Status :: 4 - Beta', ! 'Environment :: Console', ! 'License :: OSI Approved :: Python Software Foundation License', ! 'Operating System :: POSIX', ! 'Operating System :: MacOS :: MacOS X', ! 'Operating System :: Microsoft :: Windows :: Windows 95/98/2000', ! 'Operating System :: Microsoft :: Windows :: Windows NT/2000', ! 'Programming Language :: Python', ! 'Intended Audience :: End Users/Desktop', ! 'Topic :: Communications :: Email :: Filters', ! 'Topic :: Communications :: Email :: Post-Office :: POP3', ! ], ! ) From anadelonbrin at users.sourceforge.net Thu Sep 11 01:58:36 2003 From: anadelonbrin at users.sourceforge.net (Tony Meyer) Date: Thu Sep 11 03:58:40 2003 Subject: [Spambayes-checkins] website faq.txt,1.43,1.44 index.ht,1.25,1.26 Message-ID: Update of /cvsroot/spambayes/website In directory sc8-pr-cvs1:/tmp/cvs-serv22019 Modified Files: faq.txt index.ht Log Message: Add a couple of commas, and fix an unterminated section in the FAQ. Index: faq.txt =================================================================== RCS file: /cvsroot/spambayes/website/faq.txt,v retrieving revision 1.43 retrieving revision 1.44 diff -C2 -d -r1.43 -r1.44 *** faq.txt 11 Sep 2003 07:33:48 -0000 1.43 --- faq.txt 11 Sep 2003 07:58:34 -0000 1.44 *************** *** 741,745 **** `pop3.example.com` to `localhost` (or to the name of the machine you're running the proxy on) in your email client's setup, and do the same with ! your equivalent of `smtp.example.com". Hit "Get new email" and look at the headers of the emails (send yourself an email if you don't have any!) - there should be an X-Spambayes-Classification header there. It probably --- 741,745 ---- `pop3.example.com` to `localhost` (or to the name of the machine you're running the proxy on) in your email client's setup, and do the same with ! your equivalent of `smtp.example.com`. Hit "Get new email" and look at the headers of the emails (send yourself an email if you don't have any!) - there should be an X-Spambayes-Classification header there. It probably Index: index.ht =================================================================== RCS file: /cvsroot/spambayes/website/index.ht,v retrieving revision 1.25 retrieving revision 1.26 diff -C2 -d -r1.25 -r1.26 *** index.ht 8 Sep 2003 10:55:58 -0000 1.25 --- index.ht 11 Sep 2003 07:58:34 -0000 1.26 *************** *** 155,160 **** background page for a link). Rob Hooft also contributed maths/stats clues. ! Mark Hammond amazed the world with the Outlook2000 plug-in (with Tony Meyer ! Sean True and Adam Walker making significant contributions), and Richie Hindle, Neale Pickett, Tim Stone worked on the end-user applications.

        (Thanks also to Rachel Holkner for turning Anthony's gibberish into something --- 155,160 ---- background page for a link). Rob Hooft also contributed maths/stats clues. ! Mark Hammond amazed the world with the Outlook2000 plug-in (with Tony Meyer, ! Sean True, and Adam Walker making significant contributions), and Richie Hindle, Neale Pickett, Tim Stone worked on the end-user applications.

        (Thanks also to Rachel Holkner for turning Anthony's gibberish into something From sjoerd at users.sourceforge.net Thu Sep 11 02:34:44 2003 From: sjoerd at users.sourceforge.net (Sjoerd Mullender) Date: Thu Sep 11 04:34:47 2003 Subject: [Spambayes-checkins] spambayes/spambayes storage.py,1.32,1.33 Message-ID: Update of /cvsroot/spambayes/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv30693 Modified Files: storage.py Log Message: Typo in comment. Index: storage.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v retrieving revision 1.32 retrieving revision 1.33 diff -C2 -d -r1.32 -r1.33 *** storage.py 10 Sep 2003 21:08:24 -0000 1.32 --- storage.py 11 Sep 2003 08:34:42 -0000 1.33 *************** *** 142,146 **** def close(self): ! # we keep no reasources open - nothing to do pass --- 142,146 ---- def close(self): ! # we keep no resources open - nothing to do pass From richie at entrian.com Thu Sep 11 11:14:23 2003 From: richie at entrian.com (Richie Hindle) Date: Thu Sep 11 05:14:49 2003 Subject: [Spambayes-checkins] spambayes/spambayes storage.py,1.31,1.32 Message-ID: Tony, > It's a big ask to check that *every* part of the code still works. I'm not asking that people test every part of the code after making an edit. I'm asking that people *run* the specific code they've edited - that should be no kind of "ask", let alone a big one. The fact that this is one of Skip's edits we're talking about proves that no amount of expertise with a language is enough to be sure that an edit works without at least running it. It also proves that when you can't even find your toothbrush, close attention to detail can go by the wayside 8-) - how's the house move, Skip? -- Richie Hindle richie@entrian.com From skip at pobox.com Thu Sep 11 09:52:05 2003 From: skip at pobox.com (Skip Montanaro) Date: Thu Sep 11 09:52:20 2003 Subject: [Spambayes-checkins] spambayes/spambayes storage.py,1.31,1.32 In-Reply-To: <1ED4ECF91CDED24C8D012BCF2B034F13026F2927@its-xchg4.massey.ac.nz> References: <1ED4ECF91CDED24C8D012BCF2B034F1303346327@its-xchg4.massey.ac.nz> <1ED4ECF91CDED24C8D012BCF2B034F13026F2927@its-xchg4.massey.ac.nz> Message-ID: <16224.32261.481993.169764@montanaro.dyndns.org> >> Here's a thought: if people would run their edits before committing >> them, we'd have fewer bug reports. Just an idea. Tony> To be fair, I tested that the error message printed out correctly Tony> when I added it, and Skip probably checked that messages were Tony> going to stderr when he added those, just not the specific case of Tony> having no dbm modules available. Are we talking about the "import sys" which occurred after the "print >> sys.stderr, ..."? If so, I will excuse myself by stating that I always import modules at the top of a module unless it's unavoidable (as with the psycopg and MySQLdb imports in the respective SQLClassifier.load() methods, so it never even occurred to me to look for "import sys" where Richie found it. Skip From skip at pobox.com Thu Sep 11 10:06:09 2003 From: skip at pobox.com (Skip Montanaro) Date: Thu Sep 11 10:06:21 2003 Subject: [Spambayes-checkins] spambayes/spambayes storage.py,1.31,1.32 In-Reply-To: References: Message-ID: <16224.33105.483423.435036@montanaro.dyndns.org> >>>>> "Richie" == Richie Hindle writes: Richie> Tony, >> It's a big ask to check that *every* part of the code still works. Richie> I'm not asking that people test every part of the code after Richie> making an edit. I'm asking that people *run* the specific code Richie> they've edited - that should be no kind of "ask", let alone a Richie> big one. Yeah, but I run that code all the time. I filter all my mail using SpamBayes CVS. I just never have no database available, so that particular print was never executed. Richie> 8-) - how's the house move, Skip? We're out of the old house, but the new house is still a shambles, so we're bunking with my mum-in-law... Skip From montanaro at users.sourceforge.net Thu Sep 11 08:22:40 2003 From: montanaro at users.sourceforge.net (Skip Montanaro) Date: Thu Sep 11 10:22:45 2003 Subject: [Spambayes-checkins] spambayes setup.py,1.20,1.21 Message-ID: Update of /cvsroot/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv9101 Modified Files: setup.py Log Message: fix incorrect cmdclass key in setup() Index: setup.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/setup.py,v retrieving revision 1.20 retrieving revision 1.21 diff -C2 -d -r1.20 -r1.21 *** setup.py 11 Sep 2003 07:47:52 -0000 1.20 --- setup.py 11 Sep 2003 14:22:38 -0000 1.21 *************** *** 72,76 **** author_email = "spambayes@python.org", url = "http://spambayes.sourceforge.net", ! cmdclass = {'build_py': install_scripts}, scripts=['scripts/sb_client.py', 'scripts/sb_dbexpimp.py', --- 72,76 ---- author_email = "spambayes@python.org", url = "http://spambayes.sourceforge.net", ! cmdclass = {'install_scripts': install_scripts}, scripts=['scripts/sb_client.py', 'scripts/sb_dbexpimp.py', From tim_one at users.sourceforge.net Thu Sep 11 08:53:41 2003 From: tim_one at users.sourceforge.net (Tim Peters) Date: Thu Sep 11 10:53:50 2003 Subject: [Spambayes-checkins] spambayes NEWTRICKS.txt,1.1,1.2 Message-ID: Update of /cvsroot/spambayes/spambayes In directory sc8-pr-cvs1:/tmp/cvs-serv15539 Modified Files: NEWTRICKS.txt Log Message: Added comment about numeric character entities. Index: NEWTRICKS.txt =================================================================== RCS file: /cvsroot/spambayes/spambayes/NEWTRICKS.txt,v retrieving revision 1.1 retrieving revision 1.2 diff -C2 -d -r1.1 -r1.2 *** NEWTRICKS.txt 11 Sep 2003 06:57:42 -0000 1.1 --- NEWTRICKS.txt 11 Sep 2003 14:53:39 -0000 1.2 *************** *** 2,9 **** results of these ideas. ! - If the email parser was able to emit warnings about misformed MIME that it encountered, we could turn this warnings into tokens. ! - The numeric entity support looks like it will only support 8-bit characters - an obvious problem is from wacky wide-char entities. --- 2,12 ---- results of these ideas. ! - If the email parser was able to emit warnings about misformed MIME that it encountered, we could turn this warnings into tokens. ! - The numeric entity support looks like it will only support 8-bit characters - an obvious problem is from wacky wide-char entities. + [tim] It replaces those with a question mark, because chr(n) raises + an exception for n > 255, and the "except" clause returns '?' then. + Is that a problem? Probably not, for Americans and Europeans. From tim_one at users.sourceforge.net Thu Sep 11 18:22:42 2003 From: tim_one at users.sourceforge.net (Tim Peters) Date: Thu Sep 11 20:22:45 2003 Subject: [Spambayes-checkins] spambayes/Outlook2000 addin.py,1.109,1.110 Message-ID: Update of /cvsroot/spambayes/spambayes/Outlook2000 In directory sc8-pr-cvs1:/tmp/cvs-serv10021/Outlook2000 Modified Files: addin.py Log Message: ShowClues(): Add lines revealing the # ham and spam trained on. Massive imbalance is turning out to be a problem for some people, and it takes too many steps to guess that now. Index: addin.py =================================================================== RCS file: /cvsroot/spambayes/spambayes/Outlook2000/addin.py,v retrieving revision 1.109 retrieving revision 1.110 diff -C2 -d -r1.109 -r1.110 *** addin.py 8 Sep 2003 00:14:14 -0000 1.109 --- addin.py 12 Sep 2003 00:22:39 -0000 1.110 *************** *** 139,143 **** import filter filter.filter_message(msgstore_message, manager, all_actions = False) ! else: print "already was trained as good" --- 139,143 ---- import filter filter.filter_message(msgstore_message, manager, all_actions = False) ! else: print "already was trained as good" *************** *** 171,175 **** % msgstore_message.subject) return ! if HaveSeenMessage(msgstore_message, manager): # Already seen this message - user probably moving it back --- 171,175 ---- % msgstore_message.subject) return ! if HaveSeenMessage(msgstore_message, manager): # Already seen this message - user probably moving it back *************** *** 191,195 **** # rules moving the item. need_train = manager.config.filter.unsure_threshold < prop * 100 ! if need_train: TrainAsHam(msgstore_message, manager) --- 191,195 ---- # rules moving the item. need_train = manager.config.filter.unsure_threshold < prop * 100 ! if need_train: TrainAsHam(msgstore_message, manager) *************** *** 455,463 **** body = ["

        Spam Score: %d%% (%g)


        " % (round(score*100), score)] push = body.append ! # Format the clues. push("
        \n")
      push("word                                spamprob         #ham  #spam\n")
      format = " %-12g %8s %6s\n"
-     c = mgr.GetClassifier()
      fetchword = c.wordinfo.get
      for word, prob in clues:
--- 455,467 ----
      body = ["
        Spam Score: %d%% (%g)

        " % (round(score*100), score)]
      push = body.append
!     # Format the # ham and spam trained on.
!     c = mgr.GetClassifier()
      push("
        \n")
+     push("#  ham trained on: %6d\n" % c.nham)
+     push("# spam trained on: %6d\n" % c.nspam)
+     push("\n")
+     # Format the clues.
      push("word                                spamprob         #ham  #spam\n")
      format = " %-12g %8s %6s\n"
      fetchword = c.wordinfo.get
      for word, prob in clues:
***************
*** 671,675 ****
                      print "Unable to determine source folder for message '%s' - restoring to Inbox" % (subject,)
                      restore_folder = inbox_folder
!     
                  # Must train before moving, else we lose the message!
                  print "Recovering to folder '%s' and ham training message '%s' - " % (restore_folder.name, subject),
--- 675,679 ----
                      print "Unable to determine source folder for message '%s' - restoring to Inbox" % (subject,)
                      restore_folder = inbox_folder
! 
                  # Must train before moving, else we lose the message!
                  print "Recovering to folder '%s' and ham training message '%s' - " % (restore_folder.name, subject),
***************
*** 1157,1161 ****
              assert self.manager.addin is None, "Should not already have an addin"
              self.manager.addin = self
!     
              # Only now will the import of "spambayes.Version" work, as the
              # manager is what munges sys.path for us.
--- 1161,1165 ----
              assert self.manager.addin is None, "Should not already have an addin"
              self.manager.addin = self
! 
              # Only now will the import of "spambayes.Version" work, as the
              # manager is what munges sys.path for us.



From mhammond at users.sourceforge.net  Fri Sep 12 05:58:55 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Fri Sep 12 07:59:02 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py,1.6,1.7
Message-ID: 

Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv30094

Modified Files:
	pop3proxy_tray.py 
Log Message:
* Use simple logging strategy similar to the Outlook addin - if we have
  no console, redirect either to win32traceutil (source-code version) or
  to a %TEMP\SpamBayesServer1.log (yet to be released binary version).
* When running from binary, don't hack sys.path.  When running from source
  code, hack sys.path based file path rather than on os.getcwd.
* When running from binary, load the icon from the executable rather than
  a .ico file.


Index: pop3proxy_tray.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_tray.py,v
retrieving revision 1.6
retrieving revision 1.7
diff -C2 -d -r1.6 -r1.7
*** pop3proxy_tray.py	9 Sep 2003 01:07:26 -0000	1.6
--- pop3proxy_tray.py	12 Sep 2003 11:58:53 -0000	1.7
***************
*** 29,37 ****
  from win32api import *
  from win32gui import *
  
! # Allow for those without SpamBayes on the PYTHONPATH
! sys.path.insert(-1, os.getcwd())
! sys.path.insert(-1, os.path.dirname(os.getcwd()))
! sys.path.insert(-1, os.path.join(os.path.dirname(os.getcwd()),"scripts"))
  
  import sb_server
--- 29,84 ----
  from win32api import *
  from win32gui import *
+ from win32api import error as win32api_error
  
! # If we are not running in a console, redirect all print statements to the
! # win32traceutil collector.
! # You can view output either from Pythonwin's "Tools->Trace Collector Debugging Tool",
! # or simply run "win32traceutil.py" from a command prompt.
! try:
!     GetConsoleTitle()
! except win32api_error:
!     # No console - if we are running from Python sources,
!     # redirect to win32traceutil, but if running from a binary
!     # install, redirect to a log file.
!     # Want to move to logging module later, so for now, we
!     # hack together a simple logging strategy.
!     if hasattr(sys, "frozen"):
!         temp_dir = GetTempPath()
!         for i in range(3,0,-1):
!             try: os.unlink(os.path.join(temp_dir, "SpamBayesServer%d.log" % (i+1)))
!             except os.error: pass
!             try:
!                 os.rename(
!                     os.path.join(temp_dir, "SpamBayesServer%d.log" % i),
!                     os.path.join(temp_dir, "SpamBayesServer%d.log" % (i+1))
!                     )
!             except os.error: pass
!         # Open this log, as unbuffered so crashes still get written.
!         sys.stdout = open(os.path.join(temp_dir,"SpamBayesServer1.log"), "wt", 0)
!         sys.stderr = sys.stdout
!     else:
!         import win32traceutil
! 
! # Work out our "application directory", which is
! # the directory of our main .py/.exe file we
! # are running from.
! try:
!     if hasattr(sys, "frozen"):
!         if sys.frozen == "dll":
!             # Don't think we will ever run as a .DLL, but...
!             this_filename = win32api.GetModuleFileName(sys.frozendllhandle)
!         else:
!             this_filename = os.path.abspath(sys.argv[0])
!     else:
!         this_filename = os.path.abspath(__file__)
! except NameError: # no __file__
!     this_filename = os.path.abspath(sys.argv[0])
! 
! this_dir = os.path.dirname(this_filename)
! if not hasattr(sys, "frozen"):
!     # Allow for those without SpamBayes on the PYTHONPATH
!     sys.path.insert(-1, this_dir)
!     sys.path.insert(-1, os.path.dirname(this_dir))
!     sys.path.insert(-1, os.path.join(os.path.dirname(this_dir),"scripts"))
  
  import sb_server
***************
*** 47,51 ****
          # The ordering here is important - it is the order that they will
          # appear in the menu.  As dicts don't have an order, this means
!         # that the order is controlled by the id.  Any items were the
          # function is None will appear as separators.
          self.control_functions = {START_STOP_ID : ("Stop SpamBayes", self.StartStop),
--- 94,98 ----
          # The ordering here is important - it is the order that they will
          # appear in the menu.  As dicts don't have an order, this means
!         # that the order is controlled by the id.  Any items where the
          # function is None will appear as separators.
          self.control_functions = {START_STOP_ID : ("Stop SpamBayes", self.StartStop),
***************
*** 81,85 ****
  ##        iconPathName = "%s\\..\\windows\\resources\\sbicon.ico" % \
  ##                       (os.path.dirname(sb_server.__file__),)
!         if os.path.isfile(startedIconPathName) and os.path.isfile(stoppedIconPathName):
              icon_flags = win32con.LR_LOADFROMFILE | win32con.LR_DEFAULTSIZE
              self.hstartedicon = LoadImage(hinst, startedIconPathName, win32con.IMAGE_ICON, 0,
--- 128,142 ----
  ##        iconPathName = "%s\\..\\windows\\resources\\sbicon.ico" % \
  ##                       (os.path.dirname(sb_server.__file__),)
!         if hasattr(sys, "frozen"):
!             self.hstartedicon = self.hstoppedicon = None
!             hexe = GetModuleHandle(None)
!             icon_flags = win32con.LR_DEFAULTSIZE
!             self.hstartedicon = LoadImage(hexe, 1000, win32con.IMAGE_ICON, 0,
!                                           0, icon_flags)
!             self.hstopped = LoadImage(hexe, 1010, win32con.IMAGE_ICON, 0,
!                                           0, icon_flags)
!         else:
!             # If we have no icon we fail in all sorts of places - so may as
!             # well make it here :)
              icon_flags = win32con.LR_LOADFROMFILE | win32con.LR_DEFAULTSIZE
              self.hstartedicon = LoadImage(hinst, startedIconPathName, win32con.IMAGE_ICON, 0,
***************
*** 164,167 ****
--- 221,225 ----
          except KeyError:
              print "Unknown command -", id
+             return
          function()
  



From mhammond at users.sourceforge.net  Fri Sep 12 06:00:04 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Fri Sep 12 08:00:09 2003
Subject: [Spambayes-checkins] spambayes/windows/py2exe setup_all.py,1.1,1.2
Message-ID: 

Update of /cvsroot/spambayes/spambayes/windows/py2exe
In directory sc8-pr-cvs1:/tmp/cvs-serv30773

Modified Files:
	setup_all.py 
Log Message:
Add pop3proxy_tray as a GUI program.  Add the icons to the executables, and
sys.path hacks for the new world order.


Index: setup_all.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/py2exe/setup_all.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** setup_all.py	29 Aug 2003 14:44:08 -0000	1.1
--- setup_all.py	12 Sep 2003 12:00:02 -0000	1.2
***************
*** 4,21 ****
  import sys, os
  sb_top_dir = os.path.abspath(os.path.dirname(os.path.join(__file__, "../../../..")))
! try:
!     import classifier
! except ImportError:
!     sys.path.append(sb_top_dir)
! 
! try:
!     import pop3proxy_service
! except ImportError:
!     sys.path.append(os.path.join(sb_top_dir, "windows"))
!     
! try:
!     import addin
! except ImportError:
!     sys.path.append(os.path.join(sb_top_dir, "Outlook2000"))
  
  # ModuleFinder can't handle runtime changes to __path__, but win32com uses them,
--- 4,11 ----
  import sys, os
  sb_top_dir = os.path.abspath(os.path.dirname(os.path.join(__file__, "../../../..")))
! sys.path.append(sb_top_dir)
! sys.path.append(os.path.join(sb_top_dir, "windows"))
! sys.path.append(os.path.join(sb_top_dir, "scripts"))
! sys.path.append(os.path.join(sb_top_dir, "Outlook2000"))
  
  # ModuleFinder can't handle runtime changes to __path__, but win32com uses them,
***************
*** 41,44 ****
--- 31,37 ----
  class py2exe_options:
      bitmap_resources = [(1000, os.path.join(sb_top_dir, r"Outlook2000\dialogs\resources\sblogo.bmp"))]
+     icon_resources = [(1000, os.path.join(sb_top_dir, r"windows\resources\sb-started.ico")),
+                       (1010, os.path.join(sb_top_dir, r"windows\resources\sb-stopped.ico")),
+     ]
      packages = "spambayes.resources"
      excludes = "win32ui,pywin" # pywin is a package, and still seems to be included.
***************
*** 48,52 ****
  if len(sys.argv)==1:
      sys.argv = [sys.argv[0], "py2exe"]
!    
  setup(name="SpamBayes",
        packages = ["spambayes.resources"],
--- 41,45 ----
  if len(sys.argv)==1:
      sys.argv = [sys.argv[0], "py2exe"]
! 
  setup(name="SpamBayes",
        packages = ["spambayes.resources"],
***************
*** 56,59 ****
        service=["pop3proxy_service"],
        # A console exe for debugging
!       console=[os.path.join(sb_top_dir, "pop3proxy.py")],
  )
--- 49,54 ----
        service=["pop3proxy_service"],
        # A console exe for debugging
!       console=[os.path.join(sb_top_dir, "scripts", "sb_server.py")],
!       # The taskbar
!       windows=[os.path.join(sb_top_dir, "windows", "pop3proxy_tray.py")],
  )



From anadelonbrin at users.sourceforge.net  Sat Sep 13 00:03:43 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sat Sep 13 00:03:47 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_smtpproxy.py,1.1,1.2
Message-ID: 

Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv29764/scripts

Modified Files:
	sb_smtpproxy.py 
Log Message:
Fix [ 805351 ] If cc: address is not qualified, proxy fails to send message

Index: sb_smtpproxy.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_smtpproxy.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** sb_smtpproxy.py	5 Sep 2003 01:16:45 -0000	1.1
--- sb_smtpproxy.py	13 Sep 2003 04:03:41 -0000	1.2
***************
*** 310,321 ****
              return address
  
-     def splitTo(self, address):
-         """Return 'address' as undressed (host, fulladdress) tuple.
-         Handy for use with TO: addresses."""
-         start = string.index(address, '<') + 1
-         sep = string.index(address, '@') + 1
-         end = string.index(address, '>')
-         return (address[sep:end], address[start:end],)
- 
      def onTransaction(self, command, args):
          handler = self.handlers.get(command.upper(), self.onUnknown)
--- 310,313 ----
***************
*** 334,338 ****
  
      def onRcptTo(self, command, args):
!         toHost, toFull = self.splitTo(args[0])
          if toFull == options["smtpproxy", "spam_address"]:
              self.train_as_spam = True
--- 326,330 ----
  
      def onRcptTo(self, command, args):
!         toFull = self.stripAddress(args[0])
          if toFull == options["smtpproxy", "spam_address"]:
              self.train_as_spam = True



From anadelonbrin at users.sourceforge.net  Sat Sep 13 01:10:51 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sat Sep 13 01:10:55 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_server.py,1.3,1.4
Message-ID: 

Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv8944/scripts

Modified Files:
	sb_server.py 
Log Message:
It's probably a good idea to close and save the database *before* we turn it into a completely
new one...

Index: sb_server.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_server.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** sb_server.py	10 Sep 2003 04:33:17 -0000	1.3
--- sb_server.py	13 Sep 2003 05:10:49 -0000	1.4
***************
*** 714,718 ****
  def _recreateState():
      global state
-     state = State()
  
      # Close the existing listeners and create new ones.  This won't
--- 714,717 ----
***************
*** 727,730 ****
--- 726,731 ----
      state.bayes.store()
      state.bayes.close()
+ 
+     state = State()
  
      prepare(state)



From mhammond at users.sourceforge.net  Sun Sep 14 01:23:53 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Sun Sep 14 01:24:01 2003
Subject: [Spambayes-checkins] spambayes/windows/py2exe setup_all.py,1.2,1.3
Message-ID: 

Update of /cvsroot/spambayes/spambayes/windows/py2exe
In directory sc8-pr-cvs1:/tmp/cvs-serv28887

Modified Files:
	setup_all.py 
Log Message:
New the new py2exe sandbox "targets" feature I ject checked in.  This now
works for everything bar Outlook, and that is *very* close (no dialogs 
yet!)


Index: setup_all.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/py2exe/setup_all.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** setup_all.py	12 Sep 2003 12:00:02 -0000	1.2
--- setup_all.py	14 Sep 2003 05:23:51 -0000	1.3
***************
*** 17,21 ****
          modulefinder.AddPackagePath("win32com", p)
      # Not sure why this works for "win32com.mapi" for not "win32com.shell"!
!     for extra in ["win32com.shell"]:
          __import__(extra)
          m = sys.modules[extra]
--- 17,21 ----
          modulefinder.AddPackagePath("win32com", p)
      # Not sure why this works for "win32com.mapi" for not "win32com.shell"!
!     for extra in ["win32com.shell","win32com.mapi"]:
          __import__(extra)
          m = sys.modules[extra]
***************
*** 29,40 ****
  import py2exe
  
! class py2exe_options:
!     bitmap_resources = [(1000, os.path.join(sb_top_dir, r"Outlook2000\dialogs\resources\sblogo.bmp"))]
!     icon_resources = [(1000, os.path.join(sb_top_dir, r"windows\resources\sb-started.ico")),
!                       (1010, os.path.join(sb_top_dir, r"windows\resources\sb-stopped.ico")),
!     ]
!     packages = "spambayes.resources"
!     excludes = "win32ui,pywin" # pywin is a package, and still seems to be included.
  
  # Default and only distutils command is "py2exe" - save adding it to the
  # command line every single time.
--- 29,61 ----
  import py2exe
  
! class Options:
!     def __init__(self, **kw):
!         self.__dict__.update(kw)
  
+ # py2exe_options is a global name found by py2exe
+ py2exe_options = Options(
+     packages = "spambayes.resources,encodings",
+     excludes = "win32ui,pywin,pywin.debugger" # pywin is a package, and still seems to be included.
+ )
+ 
+ # These are just objects passed to py2exe
+ com_server = Options(
+     modules = ["addin"],
+     dest_base = "SpamBayes_Outlook_Addin",
+     bitmap_resources = [(1000, os.path.join(sb_top_dir, r"Outlook2000\dialogs\resources\sblogo.bmp"))],
+     create_exe = False,
+ )
+ 
+ service = Options(
+     modules = ["pop3proxy_service"]
+ )
+ sb_server = Options(
+     script = os.path.join(sb_top_dir, "scripts", "sb_server.py")
+ )
+ pop3proxy_tray = Options(
+     script = os.path.join(sb_top_dir, "windows", "pop3proxy_tray.py"),
+     icon_resources = [(1000, os.path.join(sb_top_dir, r"windows\resources\sb-started.ico")),
+                       (1010, os.path.join(sb_top_dir, r"windows\resources\sb-stopped.ico"))],
+ )
  # Default and only distutils command is "py2exe" - save adding it to the
  # command line every single time.
***************
*** 45,54 ****
        packages = ["spambayes.resources"],
        # We implement a COM object.
!       com_server=["addin"],
        # A service
!       service=["pop3proxy_service"],
        # A console exe for debugging
!       console=[os.path.join(sb_top_dir, "scripts", "sb_server.py")],
        # The taskbar
!       windows=[os.path.join(sb_top_dir, "windows", "pop3proxy_tray.py")],
  )
--- 66,75 ----
        packages = ["spambayes.resources"],
        # We implement a COM object.
!       com_server=[com_server],
        # A service
!       service=[service],
        # A console exe for debugging
!       console=[sb_server],
        # The taskbar
!       windows=[pop3proxy_tray],
  )



From tim_one at users.sourceforge.net  Sun Sep 14 16:24:15 2003
From: tim_one at users.sourceforge.net (Tim Peters)
Date: Sun Sep 14 16:24:18 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000 addin.py,1.110,1.111
Message-ID: 

Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv16908/Outlook2000

Modified Files:
	addin.py 
Log Message:
ShowClues():  Made the clue report a little prettier, and (I hope) a
little easier to follow.


Index: addin.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/addin.py,v
retrieving revision 1.110
retrieving revision 1.111
diff -C2 -d -r1.110 -r1.111
*** addin.py	12 Sep 2003 00:22:39 -0000	1.110
--- addin.py	14 Sep 2003 20:24:12 -0000	1.111
***************
*** 453,466 ****
      # way to get this as text only.  Next best then is to use HTML, 'cos at
      # least we know how to exploit it!
!     body = ["
        Spam Score: %d%% (%g)

        " % (round(score*100), score)]
      push = body.append
      # Format the # ham and spam trained on.
      c = mgr.GetClassifier()
!     push("
        \n")
!     push("#  ham trained on: %6d\n" % c.nham)
!     push("# spam trained on: %6d\n" % c.nspam)
!     push("\n")
      # Format the clues.
!     push("word                                spamprob         #ham  #spam\n")
      format = " %-12g %8s %6s\n"
      fetchword = c.wordinfo.get
--- 453,472 ----
      # way to get this as text only.  Next best then is to use HTML, 'cos at
      # least we know how to exploit it!
!     body = ["
        Combined Score: %d%% (%g)
        \n" %
!             (round(score*100), score)]
      push = body.append
+     # Format internal scores.
+     push("Internal ham score (%s): %g
        \n" % clues.pop(0))
+     push("Internal spam score (%s): %g
        \n" % clues.pop(0))
      # Format the # ham and spam trained on.
      c = mgr.GetClassifier()
!     push("
        \n")
!     push("# ham trained on: %d
        \n" % c.nham)
!     push("# spam trained on: %d
        \n" % c.nspam)
      # Format the clues.
!     push("
        %s Significant Tokens
        \n
        " % len(clues))
!     push("")
!     push("token                               spamprob         #ham  #spam\n")
!     push("")
      format = " %-12g %8s %6s\n"
      fetchword = c.wordinfo.get
***************
*** 478,482 ****
  
      # Now the raw text of the message, as best we can
!     push("
        Message Stream:

        ")
      push("
        \n")
      msg = msgstore_message.GetEmailPackageObject(strip_mime_headers=False)
--- 484,488 ----
  
      # Now the raw text of the message, as best we can
!     push("
        Message Stream
        \n")
      push("
        \n")
      msg = msgstore_message.GetEmailPackageObject(strip_mime_headers=False)
***************
*** 487,491 ****
      from spambayes.tokenizer import tokenize
      from spambayes.classifier import Set # whatever classifier uses
!     push("
        Message Tokens:

        ")
      # need to re-fetch, as the tokens we see may be different based on
      # header stripping.
--- 493,497 ----
      from spambayes.tokenizer import tokenize
      from spambayes.classifier import Set # whatever classifier uses
!     push("
        All Message Tokens
        \n")
      # need to re-fetch, as the tokens we see may be different based on
      # header stripping.
***************
*** 506,510 ****
      new_msg.Subject = "Spam Clues: " + item.Subject
      # As above, use HTMLBody else Outlook refuses to behave.
!     new_msg.HTMLBody = "" + body + ""
      # Attach the source message to it
      # Using the original message has the side-effect of marking the original
--- 512,523 ----
      new_msg.Subject = "Spam Clues: " + item.Subject
      # As above, use HTMLBody else Outlook refuses to behave.
!     new_msg.HTMLBody = """\
! 
! 
! 
! 
! """ + body + ""
      # Attach the source message to it
      # Using the original message has the side-effect of marking the original



From tim_one at users.sourceforge.net  Sun Sep 14 19:46:51 2003
From: tim_one at users.sourceforge.net (Tim Peters)
Date: Sun Sep 14 19:46:54 2003
Subject: [Spambayes-checkins] spambayes/spambayes Version.py,1.20,1.21
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv27286/spambayes

Modified Files:
	Version.py 
Log Message:
SF bug 806238:  urllib2 fails in Outlook new-version chk.
Switch to a SF URL until the problem with the spambayes.org URL is
understood.


Index: Version.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Version.py,v
retrieving revision 1.20
retrieving revision 1.21
diff -C2 -d -r1.20 -r1.21
*** Version.py	9 Sep 2003 03:09:49 -0000	1.20
--- Version.py	14 Sep 2003 23:46:49 -0000	1.21
***************
*** 11,15 ****
  """
  
! LATEST_VERSION_HOME="http://www.spambayes.org/download/Version.cfg"
  
  # This module is part of the spambayes project, which is Copyright 2002-3
--- 11,19 ----
  """
  
! # See bug 806238: urllib2 fails in Outlook new-version chk.
! # A reason for why the spambayes.org URL fails is given in a comment there.
! #LATEST_VERSION_HOME="http://www.spambayes.org/download/Version.cfg"
! # The SF URL instead works for Tim and xenogeist.
! LATEST_VERSION_HOME="http://spambayes.sourceforge.net/download/Version.cfg"
  
  # This module is part of the spambayes project, which is Copyright 2002-3
***************
*** 187,191 ****
      for appname in versions["Apps"]:
          _make_cfg_section(stream, appname, versions["Apps"][appname])
!     
  def main(args):
      if '-g' in args:
--- 191,195 ----
      for appname in versions["Apps"]:
          _make_cfg_section(stream, appname, versions["Apps"][appname])
! 
  def main(args):
      if '-g' in args:



From anadelonbrin at users.sourceforge.net  Sun Sep 14 20:37:52 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sun Sep 14 20:37:55 2003
Subject: [Spambayes-checkins] website docs.ht,1.15,1.16
Message-ID: 

Update of /cvsroot/spambayes/website
In directory sc8-pr-cvs1:/tmp/cvs-serv5196

Modified Files:
	docs.ht 
Log Message:
Fix Outlook readme and troubleshooting doc displaying in plain text in some browsers.

Index: docs.ht
===================================================================
RCS file: /cvsroot/spambayes/website/docs.ht,v
retrieving revision 1.15
retrieving revision 1.16
diff -C2 -d -r1.15 -r1.16
*** docs.ht	1 Jul 2003 00:50:11 -0000	1.15
--- docs.ht	15 Sep 2003 00:37:50 -0000	1.16
***************
*** 10,14 ****
  
      • A rudimentary set of Frequently Asked Questions (FAQ).
        • 
  
      • Instructions on integrating Spambayes into your mail system.
        • 
! 
      • The Outlook plugin includes an "About" File, and a "Troubleshooting Guide" that can be accessed via the toolbar.
  (Note that the online documentaton is always for the latest source version, and so might not correspond exactly with the version you are using.
  Always start with the documentation that came with the version you installed.)
        • 
--- 10,15 ----
  
      • A rudimentary set of Frequently Asked Questions (FAQ).
        • 
  
      • Instructions on integrating Spambayes into your mail system.
        • 
! 
      • The Outlook plugin includes an "About" File, and a 
! "Troubleshooting Guide" that can be accessed via the toolbar.
  (Note that the online documentaton is always for the latest source version, and so might not correspond exactly with the version you are using.
  Always start with the documentation that came with the version you installed.)
        • 



From anadelonbrin at users.sourceforge.net  Sun Sep 14 21:00:20 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sun Sep 14 21:00:23 2003
Subject: [Spambayes-checkins] spambayes/spambayes Version.py,1.21,1.22
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv9175/spambayes

Modified Files:
	Version.py 
Log Message:
Add a (basic) check for version option to the pop3proxy tray app.

Also add binary version info for pop3proxy.  Since pop3proxy and Outlook will be using
the same installer, should this maybe be in some central location, rather than in each
of them?

Index: Version.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Version.py,v
retrieving revision 1.21
retrieving revision 1.22
diff -C2 -d -r1.21 -r1.22
*** Version.py	14 Sep 2003 23:46:49 -0000	1.21
--- Version.py	15 Sep 2003 01:00:18 -0000	1.22
***************
*** 49,52 ****
--- 49,53 ----
          "POP3 Proxy" : {
              "Version":          0.2,
+             "BinaryVersion":    0.1,
              "Description":      "SpamBayes POP3 Proxy Beta2",
              "Date":             "September 2003",
***************
*** 55,58 ****
--- 56,66 ----
              "Full Description": """%(Description)s, version %(Version)s (%(Date)s),
  using %(InterfaceDescription)s, version %(InterfaceVersion)s""",
+             "Full Description Binary":
+                                 """%(Description)s, Binary version %(BinaryVersion)s (%(Date)s),
+ using %(InterfaceDescription)s, version %(InterfaceVersion)s""",
+             # Note this means we can change the download page later, and old
+             # versions will still go to the new page.
+             # We may also like to have a "Release Notes Page" item later?
+             "Download Page": "http://spambayes.sourceforge.net/windows.html"
          },
          "Lotus Notes Filter" : {



From anadelonbrin at users.sourceforge.net  Sun Sep 14 21:00:20 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sun Sep 14 21:00:25 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py,1.7,1.8
Message-ID: 

Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv9175/windows

Modified Files:
	pop3proxy_tray.py 
Log Message:
Add a (basic) check for version option to the pop3proxy tray app.

Also add binary version info for pop3proxy.  Since pop3proxy and Outlook will be using
the same installer, should this maybe be in some central location, rather than in each
of them?

Index: pop3proxy_tray.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_tray.py,v
retrieving revision 1.7
retrieving revision 1.8
diff -C2 -d -r1.7 -r1.8
*** pop3proxy_tray.py	12 Sep 2003 11:58:53 -0000	1.7
--- pop3proxy_tray.py	15 Sep 2003 01:00:18 -0000	1.8
***************
*** 25,28 ****
--- 25,41 ----
  import webbrowser
  import thread
+ import traceback
+ 
+ # This should just be imported from dialogs.dlgutils, but
+ # I'm not sure that we can import from the Outlook2000
+ # directory, because I don't think it gets installed.
+ ##from spambayes.Outlook2000.dialogs.dlgutils import SetWaitCursor
+ def SetWaitCursor(wait):
+     import win32gui, win32con
+     if wait:
+         hCursor = win32gui.LoadCursor(0, win32con.IDC_WAIT)
+     else:
+         hCursor = win32gui.LoadCursor(0, 0)
+     win32gui.SetCursor(hCursor)
  
  import win32con
***************
*** 100,105 ****
                                    1026 : ("View information ...", self.OpenInterface),
                                    1027 : ("Configure ...", self.OpenConfig),
                                    1028 : ("-", None),
!                                   1029 : ("Exit SpamBayes", self.OnExit),
                                    }
          message_map = {
--- 113,119 ----
                                    1026 : ("View information ...", self.OpenInterface),
                                    1027 : ("Configure ...", self.OpenConfig),
+                                   1027 : ("Check for latest version", self.CheckVersion),
                                    1028 : ("-", None),
!                                   1099 : ("Exit SpamBayes", self.OnExit),
                                    }
          message_map = {
***************
*** 260,263 ****
--- 274,312 ----
          webbrowser.open_new("http://localhost:%d/config" % \
                              (options["html_ui", "port"],))
+ 
+     def CheckVersion(self):
+         # Stolen, with few modifications, from addin.py
+         from spambayes.Version import get_version_string, \
+              get_version_number, fetch_latest_dict
+         if hasattr(sys, "frozen"):
+             version_number_key = "BinaryVersion"
+             version_string_key = "Full Description Binary"
+         else:
+             version_number_key = "Version"
+             version_string_key = "Full Description"
+ 
+         app_name = "POP3 Proxy"
+         cur_ver_string = get_version_string(app_name, version_string_key)
+         cur_ver_num = get_version_number(app_name, version_number_key)
+ 
+         try:
+             SetWaitCursor(1)
+             latest = fetch_latest_dict()
+             SetWaitCursor(0)
+             latest_ver_string = get_version_string(app_name, version_string_key,
+                                                    version_dict=latest)
+             latest_ver_num = get_version_number(app_name, version_number_key,
+                                                 version_dict=latest)
+         except:
+             print "Error checking the latest version"
+             traceback.print_exc()
+             return
+ 
+         print "Current version is %s, latest is %s." % (cur_ver_num, latest_ver_num)
+         if latest_ver_num > cur_ver_num:
+             url = get_version_string(app_name, "Download Page", version_dict=latest)
+             # Offer to open up the url
+ ##                os.startfile(url)
+       
  
  def main():



From anadelonbrin at users.sourceforge.net  Sun Sep 14 21:08:13 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sun Sep 14 21:08:17 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000
	default_bayes_customize.ini, 1.7, 1.8
Message-ID: 

Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv10633/Outlook2000

Modified Files:
	default_bayes_customize.ini 
Log Message:
Goodbye Gary-combining!  (Use the Last-Gary tag to get it back).

Index: default_bayes_customize.ini
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/default_bayes_customize.ini,v
retrieving revision 1.7
retrieving revision 1.8
diff -C2 -d -r1.7 -r1.8
*** default_bayes_customize.ini	18 Nov 2002 01:40:04 -0000	1.7
--- default_bayes_customize.ini	15 Sep 2003 01:08:11 -0000	1.8
***************
*** 4,9 ****
  
  [Tokenizer]
! # So long as Mark, Sean, and I are the primary users of this (i.e.,
! # Americans and Australians), this non-default option is very effective
  # at nailing Asian spam with little training and small database burden.
  # It should probably be exposed via the GUI, as it's not appropriate
--- 4,8 ----
  
  [Tokenizer]
! # This non-default option is very effective
  # at nailing Asian spam with little training and small database burden.
  # It should probably be exposed via the GUI, as it's not appropriate
***************
*** 24,32 ****
  
  [Classifier]
- # Uncomment the next lines if you want to use the former default for
- # scoring.
- #use_chi_squared_combining: False
- #use_gary_combining: True
- 
  # This will probably go away if testing confirms it's a Good Thing.
! experimental_ham_spam_imbalance_adjustment: True
\ No newline at end of file
--- 23,26 ----
  
  [Classifier]
  # This will probably go away if testing confirms it's a Good Thing.
! experimental_ham_spam_imbalance_adjustment: True



From anadelonbrin at users.sourceforge.net  Sun Sep 14 21:08:13 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sun Sep 14 21:08:19 2003
Subject: [Spambayes-checkins] spambayes/spambayes Options.py, 1.72,
	1.73 classifier.py, 1.9, 1.10
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv10633/spambayes

Modified Files:
	Options.py classifier.py 
Log Message:
Goodbye Gary-combining!  (Use the Last-Gary tag to get it back).

Index: Options.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v
retrieving revision 1.72
retrieving revision 1.73
diff -C2 -d -r1.72 -r1.73
*** Options.py	9 Sep 2003 07:03:54 -0000	1.72
--- Options.py	15 Sep 2003 01:08:11 -0000	1.73
***************
*** 371,382 ****
       REAL, RESTORE),
  
-     ("use_gary_combining", "Use gary-combining", False,
-      """The combining scheme currently detailed on the Robinson web page.
-      The middle ground here is touchy, varying across corpus, and within
-      a corpus across amounts of training data.  It almost never gives
-      extreme scores (near 0.0 or 1.0), but the tail ends of the ham and
-      spam distributions overlap.""",
-      BOOLEAN, RESTORE),
- 
      ("use_chi_squared_combining", "Use chi-squared combining", True,
       """For vectors of random, uniformly distributed probabilities,
--- 371,374 ----

Index: classifier.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/classifier.py,v
retrieving revision 1.9
retrieving revision 1.10
diff -C2 -d -r1.9 -r1.10
*** classifier.py	5 Sep 2003 01:15:28 -0000	1.9
--- classifier.py	15 Sep 2003 01:08:11 -0000	1.10
***************
*** 25,28 ****
--- 25,31 ----
  # advance, and it's touchy.
  #
+ # The last version of the Gary-combining scheme can be retrieved from our
+ # CVS repository via tag Last-Gary.
+ #
  # The chi-combining scheme used by default here gets closer to the theoretical
  # basis of Gary's combining scheme, and does give extreme scores, but also
***************
*** 108,181 ****
      # spamprob() implementations.  One of the following is aliased to
      # spamprob, depending on option settings.
! 
!     def gary_spamprob(self, wordstream, evidence=False):
!         """Return best-guess probability that wordstream is spam.
! 
!         wordstream is an iterable object producing words.
!         The return value is a float in [0.0, 1.0].
! 
!         If optional arg evidence is True, the return value is a pair
!             probability, evidence
!         where evidence is a list of (word, probability) pairs.
!         """
! 
!         from math import frexp
! 
!         # This combination method is due to Gary Robinson; see
!         # http://radio.weblogs.com/0101454/stories/2002/09/16/spamDetection.html
! 
!         # The real P = this P times 2**Pexp.  Likewise for Q.  We're
!         # simulating unbounded dynamic float range by hand.  If this pans
!         # out, *maybe* we should store logarithms in the database instead
!         # and just add them here.  But I like keeping raw counts in the
!         # database (they're easy to understand, manipulate and combine),
!         # and there's no evidence that this simulation is a significant
!         # expense.
!         P = Q = 1.0
!         Pexp = Qexp = 0
!         clues = self._getclues(wordstream)
!         for prob, word, record in clues:
!             P *= 1.0 - prob
!             Q *= prob
!             if P < 1e-200:  # move back into range
!                 P, e = frexp(P)
!                 Pexp += e
!             if Q < 1e-200:  # move back into range
!                 Q, e = frexp(Q)
!                 Qexp += e
! 
!         P, e = frexp(P)
!         Pexp += e
!         Q, e = frexp(Q)
!         Qexp += e
! 
!         num_clues = len(clues)
!         if num_clues:
!             #P = 1.0 - P**(1./num_clues)
!             #Q = 1.0 - Q**(1./num_clues)
!             #
!             # (x*2**e)**n = x**n * 2**(e*n)
!             n = 1.0 / num_clues
!             P = 1.0 - P**n * 2.0**(Pexp * n)
!             Q = 1.0 - Q**n * 2.0**(Qexp * n)
! 
!             # (P-Q)/(P+Q) is in -1 .. 1; scaling into 0 .. 1 gives
!             # ((P-Q)/(P+Q)+1)/2 =
!             # ((P-Q+P-Q)/(P+Q)/2 =
!             # (2*P/(P+Q)/2 =
!             # P/(P+Q)
!             prob = P/(P+Q)
!         else:
!             prob = 0.5
! 
!         if evidence:
!             clues = [(w, p) for p, w, r in clues]
!             clues.sort(lambda a, b: cmp(a[1], b[1]))
!             return prob, clues
!         else:
!             return prob
! 
!     if options["Classifier", "use_gary_combining"]:
!         spamprob = gary_spamprob
  
      # Across vectors of length n, containing random uniformly-distributed
--- 111,116 ----
      # spamprob() implementations.  One of the following is aliased to
      # spamprob, depending on option settings.
!     # Currently only chi-squared is available, but maybe there will be
!     # an alternative again someday.
  
      # Across vectors of length n, containing random uniformly-distributed



From xenogeist at users.sourceforge.net  Sun Sep 14 21:35:06 2003
From: xenogeist at users.sourceforge.net (Adam Walker)
Date: Sun Sep 14 21:35:15 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py,1.8,1.9
Message-ID: 

Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv16419/windows

Modified Files:
	pop3proxy_tray.py 
Log Message:
Support for service from Romain GUY.

Index: pop3proxy_tray.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_tray.py,v
retrieving revision 1.8
retrieving revision 1.9
diff -C2 -d -r1.8 -r1.9
*** pop3proxy_tray.py	15 Sep 2003 01:00:18 -0000	1.8
--- pop3proxy_tray.py	15 Sep 2003 01:35:04 -0000	1.9
***************
*** 43,46 ****
--- 43,47 ----
  from win32gui import *
  from win32api import error as win32api_error
+ from win32service import *
  
  # If we are not running in a console, redirect all print statements to the
***************
*** 102,105 ****
--- 103,109 ----
  
  START_STOP_ID = 1024
+ runningStatus = (SERVICE_START_PENDING, SERVICE_RUNNING, SERVICE_CONTINUE_PENDING)
+ stoppedStatus = (SERVICE_PAUSED, SERVICE_STOP_PENDING, SERVICE_STOPPED)
+ serviceName = "pop3proxy"
  
  class MainWindow(object):
***************
*** 113,118 ****
                                    1026 : ("View information ...", self.OpenInterface),
                                    1027 : ("Configure ...", self.OpenConfig),
!                                   1027 : ("Check for latest version", self.CheckVersion),
!                                   1028 : ("-", None),
                                    1099 : ("Exit SpamBayes", self.OnExit),
                                    }
--- 117,122 ----
                                    1026 : ("View information ...", self.OpenInterface),
                                    1027 : ("Configure ...", self.OpenConfig),
!                                   1028 : ("Check for latest version", self.CheckVersion),
!                                   1029 : ("-", None),
                                    1099 : ("Exit SpamBayes", self.OnExit),
                                    }
***************
*** 122,125 ****
--- 126,131 ----
              WM_TASKBAR_NOTIFY : self.OnTaskbarNotify,
          }
+         self.use_service = True
+         #self.use_service = False
  
          # Create the Window.
***************
*** 160,167 ****
  
          flags = NIF_ICON | NIF_MESSAGE | NIF_TIP
!         nid = (self.hwnd, 0, flags, WM_TASKBAR_NOTIFY, self.hstartedicon, "SpamBayes")
          Shell_NotifyIcon(NIM_ADD, nid)
          self.started = False
          self.tip = None
  
          # Start up sb_server
--- 166,187 ----
  
          flags = NIF_ICON | NIF_MESSAGE | NIF_TIP
!         nid = (self.hwnd, 0, flags, WM_TASKBAR_NOTIFY, self.hstartedicon, 
!             "SpamBayes")
          Shell_NotifyIcon(NIM_ADD, nid)
          self.started = False
          self.tip = None
+       
+         try:
+             if self.use_service and self.IsServiceAvailable():
+                 if self.GetServiceStatus() in runningStatus:
+                     self.started = True
+             else:
+                 print "Service not availible. Using thread."
+                 self.use_service = False
+                 self.started = False
+         except:
+             print "Usage of service failed. Reverting to thread."
+             self.use_service = False
+             self.started = False
  
          # Start up sb_server
***************
*** 169,182 ****
          # XXX This should determine if we are using the service, and if so
          # XXX start that, and if not kick sb_server off in a separate thread.
!         sb_server.prepare(state=sb_server.state)
!         self.StartStop()
  
      def BuildToolTip(self):
          tip = None
!         if self.started == True:
!             #%i spam %i unsure %i session %i active
!             tip = "SpamBayes %i spam %i ham %i unsure %i sessions %i active" %\
!             (sb_server.state.numSpams, sb_server.state.numHams, sb_server.state.numUnsure,
!              sb_server.state.totalSessions, sb_server.state.activeSessions)
          else:
              tip = "SpamBayes is not running"
--- 189,207 ----
          # XXX This should determine if we are using the service, and if so
          # XXX start that, and if not kick sb_server off in a separate thread.
!         if not self.use_service:
!             sb_server.prepare(state=sb_server.state)
!         if not self.started:
!             self.StartStop()
  
      def BuildToolTip(self):
          tip = None
!         if self.started:
!             if self.use_service:
!                 tip = "SpamBayes running."
!             else:
!                 tip = "SpamBayes %i spam %i ham %i unsure %i sessions %i active" %\
!                       (sb_server.state.numSpams, sb_server.state.numHams,
!                        sb_server.state.numUnsure, sb_server.state.totalSessions,
!                        sb_server.state.activeSessions)
          else:
              tip = "SpamBayes is not running"
***************
*** 184,197 ****
              
  
!     def UpdateIcon(self, hicon=None):
!         flags = NIF_TIP
!         if hicon is not None:
!             flags |= NIF_ICON
          else:
!             hicon = 0
          self.tip = self.BuildToolTip()
          nid = (self.hwnd, 0, flags, WM_TASKBAR_NOTIFY, hicon, self.tip)
          Shell_NotifyIcon(NIM_MODIFY, nid)
  
      def OnDestroy(self, hwnd, msg, wparam, lparam):
          nid = (self.hwnd, 0)
--- 209,315 ----
              
  
!     def UpdateIcon(self):
!         flags = NIF_TIP | NIF_ICON
!         if self.started:
!             hicon = self.hstartedicon 
          else:
!             hicon = self.hstoppedicon
          self.tip = self.BuildToolTip()
          nid = (self.hwnd, 0, flags, WM_TASKBAR_NOTIFY, hicon, self.tip)
          Shell_NotifyIcon(NIM_MODIFY, nid)
  
+     def IsServiceAvailable(self):
+         schSCManager = OpenSCManager(None, None, SC_MANAGER_CONNECT)
+         schService   = OpenService(schSCManager, serviceName,
+                                    SERVICE_QUERY_STATUS)
+         if schService:
+             CloseServiceHandle(schService)
+         return schService != None
+ 
+     def GetServiceStatus(self):
+         schSCManager = OpenSCManager(None, None, SC_MANAGER_CONNECT)
+         schService   = OpenService(schSCManager, serviceName,
+                                    SERVICE_QUERY_STATUS)
+         ssStatus     = QueryServiceStatus(schService)
+         CloseServiceHandle(schService)
+         return ssStatus[1]
+ 
+     def StartService(self):
+         schSCManager = OpenSCManager(None, None, SC_MANAGER_CONNECT)
+         schService   = OpenService(schSCManager, serviceName, SERVICE_START |
+                                    SERVICE_QUERY_STATUS)
+         # we assume IsServiceAvailable() was called before
+         ssStatus     = QueryServiceStatus(schService)
+         if ssStatus[1] in runningStatus:
+             self.started = True
+             CloseServiceHandle(schService)
+             return
+ 
+         StartService(schService, None)
+ 
+         ssStatus         = QueryServiceStatus(schService)
+         dwStartTickCount = GetTickCount()
+         dwOldCheckPoint  = ssStatus[5]
+ 
+         while ssStatus[1] == SERVICE_START_PENDING:
+             dwWaitTime = ssStatus[6] / 10;
+ 
+             if dwWaitTime < 1000:
+                 dwWaitTime = 1000
+             elif dwWaitTime > 10000:
+                 dwWaitTime = 10000
+     
+             Sleep(dwWaitTime);
+             ssStatus = QueryServiceStatus(schService)
+      
+             if ssStatus[5] > dwOldCheckPoint:
+                 dwStartTickCount = GetTickCount()
+                 dwOldCheckPoint = ssStatus[5]
+             else:
+                 if GetTickCount() - dwStartTickCount > ssStatus[6]:
+                     break
+ 
+         self.started = ssStatus[1] == SERVICE_RUNNING
+         CloseServiceHandle(schService)
+         self.started = True
+ 
+     def StopService(self):
+         schSCManager = OpenSCManager(None, None, SC_MANAGER_CONNECT)
+         schService   = OpenService(schSCManager, serviceName, SERVICE_STOP |
+                                    SERVICE_QUERY_STATUS)
+         # we assume IsServiceAvailable() was called before
+         ssStatus     = QueryServiceStatus(schService)
+         if ssStatus[1] in stoppedStatus:
+             self.started = False
+             CloseServiceHandle(schService)
+             return
+         
+         ControlService(schService, SERVICE_CONTROL_STOP)
+ 
+         ssStatus         = QueryServiceStatus(schService)
+         dwStartTickCount = GetTickCount()
+         dwOldCheckPoint  = ssStatus[5]
+ 
+         while ssStatus[1] == SERVICE_STOP_PENDING:
+             dwWaitTime = ssStatus[6] / 10;
+ 
+             if dwWaitTime < 1000:
+                 dwWaitTime = 1000
+             elif dwWaitTime > 10000:
+                 dwWaitTime = 10000
+     
+             Sleep(dwWaitTime);
+             ssStatus = QueryServiceStatus(schService)
+      
+             if ssStatus[5] > dwOldCheckPoint:
+                 dwStartTickCount = GetTickCount()
+                 dwOldCheckPoint = ssStatus[5]
+             else:
+                 if GetTickCount() - dwStartTickCount > ssStatus[6]:
+                     break
+ 
+         CloseServiceHandle(schService)
+         self.started = False
+         
      def OnDestroy(self, hwnd, msg, wparam, lparam):
          nid = (self.hwnd, 0)
***************
*** 239,243 ****
  
      def OnExit(self):
!         if self.started:
              sb_server.stop(sb_server.state)
              self.started = False
--- 357,361 ----
  
      def OnExit(self):
!         if self.started and not self.use_service:
              sb_server.stop(sb_server.state)
              self.started = False
***************
*** 255,278 ****
          # XXX start/stop that, and if not kick sb_server off in a separate
          # XXX thread, or stop the thread that was started.
!         if self.started:
!             sb_server.stop(sb_server.state)
!             self.started = False
!             self.control_functions[START_STOP_ID] = ("Start SpamBayes",
!                                                      self.StartStop)
!             self.UpdateIcon(self.hstoppedicon)
          else:
!             self.StartProxyThread()
              self.control_functions[START_STOP_ID] = ("Stop SpamBayes",
                                                       self.StartStop)
!             self.UpdateIcon(self.hstartedicon)
  
      def OpenInterface(self):
!         webbrowser.open_new("http://localhost:%d/" % \
!                             (options["html_ui", "port"],))
  
      def OpenConfig(self):		
!         webbrowser.open_new("http://localhost:%d/config" % \
!                             (options["html_ui", "port"],))
! 
      def CheckVersion(self):
          # Stolen, with few modifications, from addin.py
--- 373,411 ----
          # XXX start/stop that, and if not kick sb_server off in a separate
          # XXX thread, or stop the thread that was started.
!         if self.use_service:
!             if self.GetServiceStatus() in stoppedStatus:
!                 self.StartService()
!             else:
!                 self.StopService()
          else:
!             if self.started:
!                 sb_server.stop(sb_server.state)
!                 self.started = False
!             else:
!                 self.StartProxyThread()
!         self.UpdateIcon()
!         if self.started:
              self.control_functions[START_STOP_ID] = ("Stop SpamBayes",
                                                       self.StartStop)
!         else:
!             self.control_functions[START_STOP_ID] = ("Start SpamBayes",
!                                          self.StartStop)
! 
! 
  
      def OpenInterface(self):
!         if self.started:
!             webbrowser.open_new("http://localhost:%d/" % \
!                                 (options["html_ui", "port"],))
!         else:
!             self.ShowMessage("SpamBayes is not running.")
  
      def OpenConfig(self):		
!         if self.started:
!             webbrowser.open_new("http://localhost:%d/config" % \
!                                 (options["html_ui", "port"],))
!         else:
!             self.ShowMessage("SpamBayes is not running.")
!             
      def CheckVersion(self):
          # Stolen, with few modifications, from addin.py
***************
*** 299,307 ****
                                                  version_dict=latest)
          except:
!             print "Error checking the latest version"
              traceback.print_exc()
              return
  
!         print "Current version is %s, latest is %s." % (cur_ver_num, latest_ver_num)
          if latest_ver_num > cur_ver_num:
              url = get_version_string(app_name, "Download Page", version_dict=latest)
--- 432,440 ----
                                                  version_dict=latest)
          except:
!             self.ShowMessage("Error checking the latest version")
              traceback.print_exc()
              return
  
!         self.ShowMessage("Current version is %s, latest is %s." % (cur_ver_num, latest_ver_num))
          if latest_ver_num > cur_ver_num:
              url = get_version_string(app_name, "Download Page", version_dict=latest)
***************
*** 309,312 ****
--- 442,448 ----
  ##                os.startfile(url)
        
+     def ShowMessage(self, msg):
+         MessageBox(self.hwnd, msg, "SpamBayes", win32con.MB_OK)
+ 
  
  def main():



From mhammond at users.sourceforge.net  Sun Sep 14 22:10:21 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Sun Sep 14 22:10:25 2003
Subject: [Spambayes-checkins] spambayes/windows/py2exe setup_all.py,1.3,1.4
Message-ID: 

Update of /cvsroot/spambayes/spambayes/windows/py2exe
In directory sc8-pr-cvs1:/tmp/cvs-serv22215

Modified Files:
	setup_all.py 
Log Message:
More tweaks to the binary build  - correct resources in the addin,
rename the outlook DLL, etc.


Index: setup_all.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/py2exe/setup_all.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** setup_all.py	14 Sep 2003 05:23:51 -0000	1.3
--- setup_all.py	15 Sep 2003 02:10:19 -0000	1.4
***************
*** 2,6 ****
  # A distutils setup script for SpamBayes binaries
  
! import sys, os
  sb_top_dir = os.path.abspath(os.path.dirname(os.path.join(__file__, "../../../..")))
  sys.path.append(sb_top_dir)
--- 2,6 ----
  # A distutils setup script for SpamBayes binaries
  
! import sys, os, glob
  sb_top_dir = os.path.abspath(os.path.dirname(os.path.join(__file__, "../../../..")))
  sys.path.append(sb_top_dir)
***************
*** 8,11 ****
--- 8,12 ----
  sys.path.append(os.path.join(sb_top_dir, "scripts"))
  sys.path.append(os.path.join(sb_top_dir, "Outlook2000"))
+ sys.path.append(os.path.join(sb_top_dir, "Outlook2000/sandbox"))
  
  # ModuleFinder can't handle runtime changes to __path__, but win32com uses them,
***************
*** 16,20 ****
      for p in win32com.__path__[1:]:
          modulefinder.AddPackagePath("win32com", p)
-     # Not sure why this works for "win32com.mapi" for not "win32com.shell"!
      for extra in ["win32com.shell","win32com.mapi"]:
          __import__(extra)
--- 17,20 ----
***************
*** 36,49 ****
  py2exe_options = Options(
      packages = "spambayes.resources,encodings",
!     excludes = "win32ui,pywin,pywin.debugger" # pywin is a package, and still seems to be included.
  )
  
  # These are just objects passed to py2exe
! com_server = Options(
      modules = ["addin"],
!     dest_base = "SpamBayes_Outlook_Addin",
!     bitmap_resources = [(1000, os.path.join(sb_top_dir, r"Outlook2000\dialogs\resources\sblogo.bmp"))],
      create_exe = False,
  )
  
  service = Options(
--- 36,67 ----
  py2exe_options = Options(
      packages = "spambayes.resources,encodings",
!     excludes = "win32ui,pywin,pywin.debugger", # pywin is a package, and still seems to be included.
!     includes = "dialogs.resources.dialogs", # Outlook dynamic dialogs
!     dll_excludes = ["dapi.dll", "mapi32.dll"]
  )
  
+ # These must be the same IDs as in the dialogs.  We really should just extract
+ # them from our rc scripts.
+ outlook_bmp_resources = [
+     ( 125, os.path.join(sb_top_dir, r"Outlook2000\dialogs\resources\sbwizlogo.bmp")),
+     ( 127, os.path.join(sb_top_dir, r"Outlook2000\dialogs\resources\folders.bmp")),
+     (1062, os.path.join(sb_top_dir, r"Outlook2000\dialogs\resources\sblogo.bmp")),
+ ]
+ 
  # These are just objects passed to py2exe
! outlook_addin = Options(
      modules = ["addin"],
!     dest_base = "spambayes_addin",
!     bitmap_resources = outlook_bmp_resources,
      create_exe = False,
  )
+ #outlook_manager = Options(
+ #    script = os.path.join(sb_top_dir, r"Outlook2000\manager.py"),
+ #    bitmap_resources = outlook_bmp_resources,
+ #)
+ outlook_dump_props = Options(
+     script = os.path.join(sb_top_dir, r"Outlook2000\sandbox\dump_props.py"),
+     dest_base = "outlook_dump_props",
+ )
  
  service = Options(
***************
*** 58,61 ****
--- 76,80 ----
                        (1010, os.path.join(sb_top_dir, r"windows\resources\sb-stopped.ico"))],
  )
+ 
  # Default and only distutils command is "py2exe" - save adding it to the
  # command line every single time.
***************
*** 66,74 ****
        packages = ["spambayes.resources"],
        # We implement a COM object.
!       com_server=[com_server],
        # A service
        service=[service],
        # A console exe for debugging
!       console=[sb_server],
        # The taskbar
        windows=[pop3proxy_tray],
--- 85,93 ----
        packages = ["spambayes.resources"],
        # We implement a COM object.
!       com_server=[outlook_addin],
        # A service
        service=[service],
        # A console exe for debugging
!       console=[sb_server, outlook_dump_props],
        # The taskbar
        windows=[pop3proxy_tray],



From mhammond at users.sourceforge.net  Sun Sep 14 22:15:47 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Sun Sep 14 22:15:50 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py,1.9,1.10
Message-ID: 

Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv23230

Modified Files:
	pop3proxy_tray.py 
Log Message:
Correct icon loading code in the binary.


Index: pop3proxy_tray.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_tray.py,v
retrieving revision 1.9
retrieving revision 1.10
diff -C2 -d -r1.9 -r1.10
*** pop3proxy_tray.py	15 Sep 2003 01:35:04 -0000	1.9
--- pop3proxy_tray.py	15 Sep 2003 02:15:45 -0000	1.10
***************
*** 154,158 ****
              self.hstartedicon = LoadImage(hexe, 1000, win32con.IMAGE_ICON, 0,
                                            0, icon_flags)
!             self.hstopped = LoadImage(hexe, 1010, win32con.IMAGE_ICON, 0,
                                            0, icon_flags)
          else:
--- 154,158 ----
              self.hstartedicon = LoadImage(hexe, 1000, win32con.IMAGE_ICON, 0,
                                            0, icon_flags)
!             self.hstoppedicon = LoadImage(hexe, 1010, win32con.IMAGE_ICON, 0,
                                            0, icon_flags)
          else:



From mhammond at users.sourceforge.net  Mon Sep 15 00:48:52 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Mon Sep 15 00:48:56 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py,1.10,1.11
Message-ID: 

Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv15225

Modified Files:
	pop3proxy_tray.py 
Log Message:
If there was an error shutting down the proxy, just log it and continue
to exit.  The error I saw was when it got confused and wasn't even 
running, so got a "connection refused" error.
Don't terminate the GUI process via sys.exit - use PostQuitMessage instead.


Index: pop3proxy_tray.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_tray.py,v
retrieving revision 1.10
retrieving revision 1.11
diff -C2 -d -r1.10 -r1.11
*** pop3proxy_tray.py	15 Sep 2003 02:15:45 -0000	1.10
--- pop3proxy_tray.py	15 Sep 2003 04:48:50 -0000	1.11
***************
*** 358,365 ****
      def OnExit(self):
          if self.started and not self.use_service:
!             sb_server.stop(sb_server.state)
              self.started = False
          DestroyWindow(self.hwnd)
!         sys.exit()
          
      def StartProxyThread(self):
--- 358,371 ----
      def OnExit(self):
          if self.started and not self.use_service:
!             try:
!                 sb_server.stop(sb_server.state)
!             except:
!                 print "Error stopping proxy at shutdown"
!                 traceback.print_exc()
!                 print "Shutting down anyway..."
! 
              self.started = False
          DestroyWindow(self.hwnd)
!         PostQuitMessage(0)
          
      def StartProxyThread(self):



From mhammond at users.sourceforge.net  Mon Sep 15 01:37:04 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Mon Sep 15 01:37:09 2003
Subject: [Spambayes-checkins] spambayes/windows/py2exe setup_all.py,1.4,1.5
Message-ID: 

Update of /cvsroot/spambayes/spambayes/windows/py2exe
In directory sc8-pr-cvs1:/tmp/cvs-serv23882

Modified Files:
	setup_all.py 
Log Message:
Split the apps into 2 directories, plus a 'lib' directory.


Index: setup_all.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/py2exe/setup_all.py,v
retrieving revision 1.4
retrieving revision 1.5
diff -C2 -d -r1.4 -r1.5
*** setup_all.py	15 Sep 2003 02:10:19 -0000	1.4
--- setup_all.py	15 Sep 2003 05:37:02 -0000	1.5
***************
*** 38,42 ****
      excludes = "win32ui,pywin,pywin.debugger", # pywin is a package, and still seems to be included.
      includes = "dialogs.resources.dialogs", # Outlook dynamic dialogs
!     dll_excludes = ["dapi.dll", "mapi32.dll"]
  )
  
--- 38,43 ----
      excludes = "win32ui,pywin,pywin.debugger", # pywin is a package, and still seems to be included.
      includes = "dialogs.resources.dialogs", # Outlook dynamic dialogs
!     dll_excludes = ["dapi.dll", "mapi32.dll"],
!     lib_dir = "lib",
  )
  
***************
*** 47,50 ****
--- 48,54 ----
      ( 127, os.path.join(sb_top_dir, r"Outlook2000\dialogs\resources\folders.bmp")),
      (1062, os.path.join(sb_top_dir, r"Outlook2000\dialogs\resources\sblogo.bmp")),
+     # and these are currently hard-coded in addin.py
+     (6000, os.path.join(sb_top_dir, r"Outlook2000\images\recover_ham.bmp")),
+     (6001, os.path.join(sb_top_dir, r"Outlook2000\images\delete_as_spam.bmp")),
  ]
  
***************
*** 52,56 ****
  outlook_addin = Options(
      modules = ["addin"],
!     dest_base = "spambayes_addin",
      bitmap_resources = outlook_bmp_resources,
      create_exe = False,
--- 56,60 ----
  outlook_addin = Options(
      modules = ["addin"],
!     dest_base = "outlook/spambayes_addin",
      bitmap_resources = outlook_bmp_resources,
      create_exe = False,
***************
*** 62,75 ****
  outlook_dump_props = Options(
      script = os.path.join(sb_top_dir, r"Outlook2000\sandbox\dump_props.py"),
!     dest_base = "outlook_dump_props",
  )
  
  service = Options(
      modules = ["pop3proxy_service"]
  )
  sb_server = Options(
      script = os.path.join(sb_top_dir, "scripts", "sb_server.py")
  )
  pop3proxy_tray = Options(
      script = os.path.join(sb_top_dir, "windows", "pop3proxy_tray.py"),
      icon_resources = [(1000, os.path.join(sb_top_dir, r"windows\resources\sb-started.ico")),
--- 66,82 ----
  outlook_dump_props = Options(
      script = os.path.join(sb_top_dir, r"Outlook2000\sandbox\dump_props.py"),
!     dest_base = "outlook/outlook_dump_props",
  )
  
  service = Options(
+     dest_base = "proxy/pop3proxy_service",
      modules = ["pop3proxy_service"]
  )
  sb_server = Options(
+     dest_base = "proxy/sb_server",
      script = os.path.join(sb_top_dir, "scripts", "sb_server.py")
  )
  pop3proxy_tray = Options(
+     dest_base = "proxy/pop3proxy_tray",
      script = os.path.join(sb_top_dir, "windows", "pop3proxy_tray.py"),
      icon_resources = [(1000, os.path.join(sb_top_dir, r"windows\resources\sb-started.ico")),
***************
*** 77,80 ****
--- 84,93 ----
  )
  
+ outlook_doc_files = [
+     ["outlook", [os.path.join(sb_top_dir, r"Outlook2000\about.html")]],
+     ["outlook/docs", glob.glob(os.path.join(sb_top_dir, r"Outlook2000\docs\*.html"))],
+     ["outlook/docs/images", glob.glob(os.path.join(sb_top_dir, r"Outlook2000\docs\images\*.jpg"))],
+ ]
+ 
  # Default and only distutils command is "py2exe" - save adding it to the
  # command line every single time.
***************
*** 92,94 ****
--- 105,109 ----
        # The taskbar
        windows=[pop3proxy_tray],
+       # and the misc data files
+       data_files = outlook_doc_files,
  )



From mhammond at users.sourceforge.net  Mon Sep 15 02:25:35 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Mon Sep 15 02:25:40 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000 msgstore.py,1.75,1.76
Message-ID: 

Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv32602

Modified Files:
	msgstore.py 
Log Message:
* Handle exceptions getting the "receive folders".
* Add a __hash__ method that does the right thing for message objects.


Index: msgstore.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/msgstore.py,v
retrieving revision 1.75
retrieving revision 1.76
diff -C2 -d -r1.75 -r1.76
*** msgstore.py	7 Sep 2003 23:43:21 -0000	1.75
--- msgstore.py	15 Sep 2003 06:25:33 -0000	1.76
***************
*** 370,373 ****
--- 370,378 ----
                  hex_folder_eid = mapi.HexFromBin(folder_eid)
                  hex_store_eid = mapi.HexFromBin(store_eid)
+             except pythoncom.com_error, details:
+                 if not IsNotAvailableCOMException(details):
+                     print "ERROR enumerating a receive folder -", details
+                 continue
+             try:
                  folder = self.GetFolder((hex_store_eid, hex_folder_eid))
                  # For 'unconfigured' stores, or "stand-alone" PST files,
***************
*** 376,383 ****
                  if folder.GetParent() is not None:
                      yield folder
!             except pythoncom.com_error, details:
!                 if not IsNotAvailableCOMException(details):
!                     print "ERROR enumerating a receive folder -", details
                  # but we just continue
  
  _MapiTypeMap = {
--- 381,388 ----
                  if folder.GetParent() is not None:
                      yield folder
!             except MsgStoreException, details:
!                 print "ERROR opening receive folder -", details
                  # but we just continue
+                 continue
  
  _MapiTypeMap = {
***************
*** 687,695 ****
                                       id_str)
  
      def __eq__(self, other):
-         if other is None: return False
          ceid = self.msgstore.session.CompareEntryIDs
!         return ceid(self.id[0], other.id[0]) and \
!                ceid(self.id[1], other.id[1])
  
      def __ne__(self, other):
--- 692,707 ----
                                       id_str)
  
+     # as per search-key comments above, we also "enforce" this at the Python
+     # level.  2 different messages, but one copied from the other, will
+     # return "==".
+     # Not being consistent could cause subtle bugs, especially in interactions
+     # with various test tools.
+     # Compare the GetID() results if you need to know different messages.
+     def __hash__(self):
+         return hash(self.searchkey)
+ 
      def __eq__(self, other):
          ceid = self.msgstore.session.CompareEntryIDs
!         return ceid(self.searchkey, other.searchkey)
  
      def __ne__(self, other):



From mhammond at users.sourceforge.net  Mon Sep 15 02:26:38 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Mon Sep 15 02:26:41 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000 addin.py,1.111,1.112
Message-ID: 

Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv321

Modified Files:
	addin.py 
Log Message:
Support for py2exe - add a "DllRegisterServer" method, and load button
bitmaps from the DLL rather than the source .bmp in binary builds.


Index: addin.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/addin.py,v
retrieving revision 1.111
retrieving revision 1.112
diff -C2 -d -r1.111 -r1.112
*** addin.py	14 Sep 2003 20:24:12 -0000	1.111
--- addin.py	15 Sep 2003 06:26:35 -0000	1.112
***************
*** 1,4 ****
  # SpamBayes Outlook Addin
- 
  import sys, os
  import warnings
--- 1,3 ----
***************
*** 721,732 ****
      # Also note that the clipboard takes ownership of the handle -
      # thus, we can not simply perform this load once and reuse the image.
!     if not os.path.isabs(fname):
!         # images relative to the application path
!         fname = os.path.join(manager.application_directory,
!                                  "images", fname)
!     if not os.path.isfile(fname):
!         print "WARNING - Trying to use image '%s', but it doesn't exist" % (fname,)
!         return None
!     handle = win32gui.LoadImage(0, fname, win32con.IMAGE_BITMAP, 0, 0, win32con.LR_DEFAULTSIZE | win32con.LR_LOADFROMFILE)
      win32clipboard.OpenClipboard()
      win32clipboard.SetClipboardData(win32con.CF_BITMAP, handle)
--- 720,741 ----
      # Also note that the clipboard takes ownership of the handle -
      # thus, we can not simply perform this load once and reuse the image.
!     # Hacks for the binary - we can get the bitmaps from resources.
!     if hasattr(sys, "frozen"):
!         if fname=="recover_ham.bmp":
!             bid = 6000
!         elif fname=="delete_as_spam.bmp":
!             bid = 6001
!         else:
!             raise RuntimeError, "What bitmap to use for '%s'?" % fname
!         handle = win32gui.LoadImage(sys.frozendllhandle, bid, win32con.IMAGE_BITMAP, 0, 0, win32con.LR_DEFAULTSIZE)
!     else:
!         if not os.path.isabs(fname):
!             # images relative to the application path
!             fname = os.path.join(manager.application_directory,
!                                      "images", fname)
!         if not os.path.isfile(fname):
!             print "WARNING - Trying to use image '%s', but it doesn't exist" % (fname,)
!             return None
!         handle = win32gui.LoadImage(0, fname, win32con.IMAGE_BITMAP, 0, 0, win32con.LR_DEFAULTSIZE | win32con.LR_LOADFROMFILE)
      win32clipboard.OpenClipboard()
      win32clipboard.SetClipboardData(win32con.CF_BITMAP, handle)
***************
*** 1407,1415 ****
          pass
  
  if __name__ == '__main__':
      import win32com.server.register
      win32com.server.register.UseCommandLine(OutlookAddin)
      if "--unregister" in sys.argv:
!         UnregisterAddin(OutlookAddin)
      else:
!         RegisterAddin(OutlookAddin)
--- 1416,1430 ----
          pass
  
+ def DllRegisterServer():
+     RegisterAddin(OutlookAddin)
+ 
+ def DllUnregisterServer():
+     UnregisterAddin(OutlookAddin)
+ 
  if __name__ == '__main__':
      import win32com.server.register
      win32com.server.register.UseCommandLine(OutlookAddin)
      if "--unregister" in sys.argv:
!         DllUnregisterServer()
      else:
!         DllRegisterServer()



From mhammond at users.sourceforge.net  Mon Sep 15 02:27:11 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Mon Sep 15 02:27:14 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs/resources
	__init__.py, 1.3, 1.4
Message-ID: 

Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs/resources
In directory sc8-pr-cvs1:/tmp/cvs-serv461/resources

Modified Files:
	__init__.py 
Log Message:
Load dialog bitmaps directly from the executable in binary builds.


Index: __init__.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/resources/__init__.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** __init__.py	5 Sep 2003 06:53:24 -0000	1.3
--- __init__.py	15 Sep 2003 06:27:09 -0000	1.4
***************
*** 6,9 ****
--- 6,10 ----
      if type(bmpid)==type(0):
          bmpid = rc_parser.names[bmpid]
+     int_bmpid = rc_parser.ids[bmpid]
      # For both binary and source versions, we currently load from files.
      # In future py2exe built binary versions we will be able to load the
***************
*** 11,22 ****
      filename = rc_parser.bitmaps[bmpid]
      if hasattr(sys, "frozen"):
!         # bitmap in the app/images directory
!         # dont have manager available :(
!         dll_filename = win32api.GetModuleFileName(sys.frozendllhandle)
!         app_dir = os.path.dirname(dll_filename)
!         filename = os.path.join(app_dir, "images", filename)
      else:
          if not os.path.isabs(filename):
              # In this directory
              filename = os.path.join( os.path.dirname( __file__ ), filename)
!     return 0, filename, win32con.LR_LOADFROMFILE
--- 12,26 ----
      filename = rc_parser.bitmaps[bmpid]
      if hasattr(sys, "frozen"):
!         # in our .exe/.dll - load from that.
!         if sys.frozen=="dll":
!             hmod = sys.frozendllhandle
!         else:
!             hmod = win32api.GetModuleHandle(None)
!         return hmod, int_bmpid, 0
      else:
+         # source code - load the .bmp directly.
          if not os.path.isabs(filename):
              # In this directory
              filename = os.path.join( os.path.dirname( __file__ ), filename)
!         return 0, filename, win32con.LR_LOADFROMFILE
!     assert 0, "not reached"



From kennypitt at hotmail.com  Mon Sep 15 10:22:29 2003
From: kennypitt at hotmail.com (Kenny Pitt)
Date: Mon Sep 15 10:22:52 2003
Subject: [Spambayes-checkins] spambayes/spambayes Options.py, 1.72,
	1.73 classifier.py, 1.9, 1.10
In-Reply-To: 
Message-ID: <000001c37b94$cd683240$300a10ac@spidynamics.com>

Tony Meyer wrote:
> Update of /cvsroot/spambayes/spambayes/spambayes
> In directory sc8-pr-cvs1:/tmp/cvs-serv10633/spambayes
> 
> Modified Files:
> 	Options.py classifier.py
> Log Message:
> Goodbye Gary-combining!  (Use the Last-Gary tag to get it back).
> 

Should we also drop the 'use_chi_squared_combining" option?  Currently,
setting 'use_chi_squared_combining' to false leaves you with no
Classifier.spamprob() implementation.

-- 
Kenny Pitt


From anadelonbrin at users.sourceforge.net  Mon Sep 15 18:41:34 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Mon Sep 15 18:41:38 2003
Subject: [Spambayes-checkins] spambayes NEWTRICKS.txt,1.2,1.3
Message-ID: 

Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv25878

Modified Files:
	NEWTRICKS.txt 
Log Message:
Ideas that it would be nice to try at some point.

Index: NEWTRICKS.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/NEWTRICKS.txt,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** NEWTRICKS.txt	11 Sep 2003 14:53:39 -0000	1.2
--- NEWTRICKS.txt	15 Sep 2003 22:41:31 -0000	1.3
***************
*** 11,12 ****
--- 11,21 ----
    an exception for n > 255, and the "except" clause returns '?' then.
    Is that a problem?  Probably not, for Americans and Europeans.
+ 
+ - The ratio of upper-case to lower-case characters in an entire message.
+   (I'm not sure how expensive it would be to calculate this).  A lot
+   of the Nigerian scam spams I get are predominately upper-case, and
+   none of my ham.
+ 
+ - A token indicating the length of a message (rounded to an appropriate
+   level.  Also a token indicating the ratio of message length to the
+   number of tokens, and a token indicating the number of tokens.



From anadelonbrin at users.sourceforge.net  Tue Sep 16 00:42:34 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Tue Sep 16 00:46:11 2003
Subject: [Spambayes-checkins] spambayes/spambayes FileCorpus.py,1.5,1.6
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv25187/spambayes

Modified Files:
	FileCorpus.py 
Log Message:
Fix [ 795145 ] pop3proxy review page dies with mixed gzip/non messages

Index: FileCorpus.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/FileCorpus.py,v
retrieving revision 1.5
retrieving revision 1.6
diff -C2 -d -r1.5 -r1.6
*** FileCorpus.py	5 May 2003 01:22:11 -0000	1.5
--- FileCorpus.py	16 Sep 2003 04:42:32 -0000	1.6
***************
*** 207,223 ****
      def load(self):
          '''Read the Message substance from the file'''
! 
          if options["globals", "verbose"]:
              print 'loading', self.file_name
  
          pn = self.pathname()
          try:
!             fp = open(pn, 'rb')
          except IOError, e:
              if e.errno != errno.ENOENT:
                  raise
          else:
!             self.setSubstance(fp.read())
!             fp.close()
  
      def store(self):
--- 207,246 ----
      def load(self):
          '''Read the Message substance from the file'''
!         # This is a tricky one!  Some people might have a combination
!         # of gzip and non-gzip messages, especially when they first
!         # change to or from gzip.  They should be able to see (but
!         # not create) either type, so a FileMessage load needs to be
!         # able to load gzip messages, even though it is a FileMessage
!         # subclass (GzipFileMessage) that adds the ability to store
!         # messages gzipped.  If someone can think of a classier (pun
!         # intended) way of doing this, be my guest.
          if options["globals", "verbose"]:
              print 'loading', self.file_name
  
          pn = self.pathname()
+ 
          try:
!             fp = gzip.open(pn, 'rb')
          except IOError, e:
              if e.errno != errno.ENOENT:
                  raise
          else:
!             try:
!                 self.setSubstance(fp.read())
!             except IOError, e:
!                 if str(e) == 'Not a gzipped file':
!                     # We've probably got both gzipped messages and
!                     # non-gzipped messages, and need to work with both.
!                     fp.close()
!                     try:
!                         fp = open(self.pathname(), 'rb')
!                     except IOError, e:
!                         if e.errno != errno.ENOENT:
!                             raise
!                     else:
!                         self.setSubstance(fp.read())
!                         fp.close()
!             else:
!                 fp.close()
  
      def store(self):
***************
*** 299,321 ****
  class GzipFileMessage(FileMessage):
      '''Message that persists as a zipped file system artifact.'''
- 
-     def load(self):
-         '''Read the Message substance from the file'''
- 
-         if options["globals", "verbose"]:
-             print 'loading', self.file_name
- 
-         pn = self.pathname()
- 
-         try:
-             fp = gzip.open(pn, 'rb')
-         except IOError, e:
-             if e.errno != errno.ENOENT:
-                 raise
-         else:
-             self.setSubstance(fp.read())
-             fp.close()
- 
- 
      def store(self):
          '''Write the Message substance to the file'''
--- 322,325 ----



From mhammond at users.sourceforge.net  Tue Sep 16 01:21:05 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Tue Sep 16 01:21:08 2003
Subject: [Spambayes-checkins] spambayes/windows/py2exe setup_all.py,1.5,1.6
Message-ID: 

Update of /cvsroot/spambayes/spambayes/windows/py2exe
In directory sc8-pr-cvs1:/tmp/cvs-serv31892

Modified Files:
	setup_all.py 
Log Message:
Add typelibs.


Index: setup_all.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/py2exe/setup_all.py,v
retrieving revision 1.5
retrieving revision 1.6
diff -C2 -d -r1.5 -r1.6
*** setup_all.py	15 Sep 2003 05:37:02 -0000	1.5
--- setup_all.py	16 Sep 2003 05:21:03 -0000	1.6
***************
*** 40,43 ****
--- 40,48 ----
      dll_excludes = ["dapi.dll", "mapi32.dll"],
      lib_dir = "lib",
+     typelibs = [
+         ('{00062FFF-0000-0000-C000-000000000046}', 0, 9, 0),
+         ('{2DF8D04C-5BFA-101B-BDE5-00AA0044DE52}', 0, 2, 1),
+         ('{AC0714F2-3D04-11D1-AE7D-00A0C90F26F4}', 0, 1, 0),
+     ]
  )
  
***************
*** 101,105 ****
        # A service
        service=[service],
!       # A console exe for debugging
        console=[sb_server, outlook_dump_props],
        # The taskbar
--- 106,110 ----
        # A service
        service=[service],
!       # console exes for debugging
        console=[sb_server, outlook_dump_props],
        # The taskbar



From mhammond at users.sourceforge.net  Tue Sep 16 10:00:16 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Tue Sep 16 10:00:20 2003
Subject: [Spambayes-checkins] spambayes/spambayes Options.py,1.73,1.74
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv375

Modified Files:
	Options.py 
Log Message:
If we are running Windows, have no valid config file specified, and have
the win32all extensions available, default to:
\Documents and Settings\[user]\Application Data\SpamBayes\Proxy


Index: Options.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v
retrieving revision 1.73
retrieving revision 1.74
diff -C2 -d -r1.73 -r1.74
*** Options.py	15 Sep 2003 01:08:11 -0000	1.73
--- Options.py	16 Sep 2003 14:00:12 -0000	1.74
***************
*** 18,22 ****
  """
  
! import os
  
  __all__ = ['options']
--- 18,22 ----
  """
  
! import sys, os
  
  __all__ = ['options']
***************
*** 976,977 ****
--- 976,995 ----
  if not optionsPathname:
      optionsPathname = os.path.abspath('bayescustomize.ini')
+     if sys.platform.startswith("win") and not os.path.isfile(optionsPathname):
+         # If we are on Windows and still don't have an INI, default to the
+         # 'per-user' directory.
+         try:
+             from win32com.shell import shell, shellcon
+             windowsUserDirectory = os.path.join(
+                     shell.SHGetFolderPath(0,shellcon.CSIDL_APPDATA,0,0),
+                     "SpamBayes", "Proxy")
+             optionsPathname = os.path.join(windowsUserDirectory, 'bayescustomize.ini')
+         except ImportError:
+             # We are on Windows, with no BAYESCUSTOMIZE set, no ini file
+             # in the current directory, and no win32 extensions installed
+             # to locate the "user" directory - seeing things are so lamely
+             # setup, it is worth printing a warning
+             print "NOTE: We can not locate an INI file for SpamBayes, and the"
+             print "Python for Windows extensions are not installed, meaning we"
+             print "can't locate your 'user' directory.  An empty configuration"
+             print "file at '%s' will be used." % optionsPathname.encode('mbcs')



From mhammond at users.sourceforge.net  Tue Sep 16 10:01:49 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Tue Sep 16 10:01:55 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_service.py, 1.11,
	1.12
Message-ID: 

Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv1252

Modified Files:
	pop3proxy_service.py 
Log Message:
Only munge sys.path in source-code versions, and as the service starts,
have it report the username and ini file it is using.
In binary builds, write a log to %temp%\SpamBayesServicen.log


Index: pop3proxy_service.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_service.py,v
retrieving revision 1.11
retrieving revision 1.12
diff -C2 -d -r1.11 -r1.12
*** pop3proxy_service.py	9 Sep 2003 01:26:16 -0000	1.11
--- pop3proxy_service.py	16 Sep 2003 14:01:47 -0000	1.12
***************
*** 18,45 ****
  
  import sys, os
! # We are in the 'spambayes\win32' directory.  We
! # need the parent on sys.path, so 'spambayes.spambayes' is a package,
! # and 'pop3proxy' is a module
  try:
!     # module imported by service manager, or 2.3 (in which __main__
!     # exists, *and* sys.argv[0] is always already absolute)
!     this_filename=__file__
! except NameError:
!     this_filename = sys.argv[0]
! if not os.path.isabs(sys.argv[0]):
!     # Python 2.3 __main__
!     # patch up sys.argv, as our cwd will confuse service registration code
!     sys.argv[0] = os.path.abspath(sys.argv[0])
!     this_filename = sys.argv[0]
  
! print sys.argv[0]
! sb_dir = os.path.dirname(os.path.dirname(this_filename))
! sb_scripts_dir = os.path.join(sb_dir,"scripts")
  
! sys.path.insert(0, sb_dir)
! sys.path.insert(-1, sb_scripts_dir)
! # and change directory here, so pop3proxy uses the default
! # config file etc
! os.chdir(sb_dir)
  
  # Rest of the standard Python modules we use.
--- 18,76 ----
  
  import sys, os
! 
! # Messages from pop3proxy will go nowhere when executed as a service
! # Try and detect that print will go nowhere and redirect.
! # redirect output somewhere useful when running as a service.
! import win32api
  try:
!     win32api.GetConsoleTitle()
! except win32api.error:
!     # No console - if we are running from Python sources,
!     # redirect to win32traceutil, but if running from a binary
!     # install, redirect to a log file.
!     # Want to move to logging module later, so for now, we
!     # hack together a simple logging strategy.
!     if hasattr(sys, "frozen"):
!         temp_dir = win32api.GetTempPath()
!         for i in range(3,0,-1):
!             try: os.unlink(os.path.join(temp_dir, "SpamBayesService%d.log" % (i+1)))
!             except os.error: pass
!             try:
!                 os.rename(
!                     os.path.join(temp_dir, "SpamBayesService%d.log" % i),
!                     os.path.join(temp_dir, "SpamBayesService%d.log" % (i+1))
!                     )
!             except os.error: pass
!         # Open this log, as unbuffered so crashes still get written.
!         sys.stdout = open(os.path.join(temp_dir,"SpamBayesService1.log"), "wt", 0)
!         sys.stderr = sys.stdout
!     else:
!         import win32traceutil
  
! # If running from sources, patch up sys.path
! if not hasattr(sys, "frozen"):
!     # We are in the 'spambayes\win32' directory.  We
!     # need the parent on sys.path, so 'spambayes.spambayes' is a package,
!     # and 'pop3proxy' is a module
!     try:
!         # module imported by service manager, or 2.3 (in which __main__
!         # exists, *and* sys.argv[0] is always already absolute)
!         this_filename=__file__
!     except NameError:
!         this_filename = sys.argv[0]
!     if not os.path.isabs(sys.argv[0]):
!         # Python 2.3 __main__
!         # patch up sys.argv, as our cwd will confuse service registration code
!         sys.argv[0] = os.path.abspath(sys.argv[0])
!         this_filename = sys.argv[0]
  
!     sb_dir = os.path.dirname(os.path.dirname(this_filename))
!     sb_scripts_dir = os.path.join(sb_dir,"scripts")
!     
!     sys.path.insert(0, sb_dir)
!     sys.path.insert(-1, sb_scripts_dir)
!     # and change directory here, so pop3proxy uses the default
!     # config file etc
!     os.chdir(sb_dir)
  
  # Rest of the standard Python modules we use.
***************
*** 54,75 ****
  import win32serviceutil, win32service
  import pywintypes, win32con, winerror
- 
  from ntsecuritycon import *
  
- # Messages from pop3proxy will go nowhere when executed as a service
- # Try and detect that print will go nowhere and redirect.
- try:
-     # redirect output somewhere useful when running as a service.
-     import win32api
-     try:
-         win32api.GetConsoleTitle()
-     except win32api.error:
-         # no console - import win32traceutil
-         import win32traceutil
-         print "popproxy service module loading (as user %s)..." \
-                 % win32api.GetUserName()
- except ImportError:
-     pass
- 
  class Service(win32serviceutil.ServiceFramework):
      # The script name was changed to "sb_server" but I'll leave this as pop3proxy
--- 85,90 ----
***************
*** 88,92 ****
          self.event_stopping.set()
          sb_server.stop(sb_server.state)        
-         
  
      def SvcDoRun(self):
--- 103,106 ----
***************
*** 101,109 ****
          # Write an event log record - in debug mode we will also 
          # see this message printed.
          import servicemanager
          servicemanager.LogMsg(
              servicemanager.EVENTLOG_INFORMATION_TYPE,
              servicemanager.PYS_SERVICE_STARTED,
!             (self._svc_name_, '')
              )
  
--- 115,127 ----
          # Write an event log record - in debug mode we will also 
          # see this message printed.
+         from spambayes.Options import optionsPathname
+         extra = " as user '%s', using config file '%s'" \
+                 % (win32api.GetUserNameEx(2).encode("mbcs"),
+                    optionsPathname.encode("mbcs"))
          import servicemanager
          servicemanager.LogMsg(
              servicemanager.EVENTLOG_INFORMATION_TYPE,
              servicemanager.PYS_SERVICE_STARTED,
!             (self._svc_name_, extra)
              )
  



From mhammond at users.sourceforge.net  Tue Sep 16 10:13:12 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Tue Sep 16 10:13:20 2003
Subject: [Spambayes-checkins] spambayes/spambayes Options.py,1.74,1.75
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv3835

Modified Files:
	Options.py 
Log Message:
Might as well beat our users to testing with usernames/paths that include
extended characters.
While I was at it, might as well ensure the "per user" directory exists!


Index: Options.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v
retrieving revision 1.74
retrieving revision 1.75
diff -C2 -d -r1.74 -r1.75
*** Options.py	16 Sep 2003 14:00:12 -0000	1.74
--- Options.py	16 Sep 2003 14:13:08 -0000	1.75
***************
*** 984,988 ****
                      shell.SHGetFolderPath(0,shellcon.CSIDL_APPDATA,0,0),
                      "SpamBayes", "Proxy")
!             optionsPathname = os.path.join(windowsUserDirectory, 'bayescustomize.ini')
          except ImportError:
              # We are on Windows, with no BAYESCUSTOMIZE set, no ini file
--- 984,997 ----
                      shell.SHGetFolderPath(0,shellcon.CSIDL_APPDATA,0,0),
                      "SpamBayes", "Proxy")
!             try:
!                 if not os.path.isdir(windowsUserDirectory):
!                     os.makedirs(windowsUserDirectory)
!                 optionsPathname = os.path.join(windowsUserDirectory,
!                                                'bayescustomize.ini')
!                 # Not everyone is unicode aware - keep it a string.
!                 optionsPathname = optionsPathname.encode("mbcs")
!             except os.error:
!                 # unable to make the directory - stick to default.
!                 pass
          except ImportError:
              # We are on Windows, with no BAYESCUSTOMIZE set, no ini file



From mhammond at users.sourceforge.net  Tue Sep 16 10:14:13 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Tue Sep 16 10:14:16 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_service.py, 1.12,
	1.13
Message-ID: 

Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv4080

Modified Files:
	pop3proxy_service.py 
Log Message:
Pathname will always be a string (rather than Unicode) for the forseeable 
future in the proxy.


Index: pop3proxy_service.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_service.py,v
retrieving revision 1.12
retrieving revision 1.13
diff -C2 -d -r1.12 -r1.13
*** pop3proxy_service.py	16 Sep 2003 14:01:47 -0000	1.12
--- pop3proxy_service.py	16 Sep 2003 14:14:11 -0000	1.13
***************
*** 118,122 ****
          extra = " as user '%s', using config file '%s'" \
                  % (win32api.GetUserNameEx(2).encode("mbcs"),
!                    optionsPathname.encode("mbcs"))
          import servicemanager
          servicemanager.LogMsg(
--- 118,122 ----
          extra = " as user '%s', using config file '%s'" \
                  % (win32api.GetUserNameEx(2).encode("mbcs"),
!                    optionsPathname)
          import servicemanager
          servicemanager.LogMsg(



From anadelonbrin at users.sourceforge.net  Tue Sep 16 19:51:14 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Tue Sep 16 19:51:24 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs dialog_map.py,
	1.33, 1.34
Message-ID: 

Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs
In directory sc8-pr-cvs1:/tmp/cvs-serv30309/Outlook2000/dialogs

Modified Files:
	dialog_map.py 
Log Message:
Add a warning for those with highly (>5 times) imbalanced ham and spam.

** I don't have something to tinker with the dialog layout - could someone that does please
make room for the error message? **

Index: dialog_map.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/dialog_map.py,v
retrieving revision 1.33
retrieving revision 1.34
diff -C2 -d -r1.33 -r1.34
*** dialog_map.py	5 Sep 2003 11:50:57 -0000	1.33
--- dialog_map.py	16 Sep 2003 23:51:12 -0000	1.34
***************
*** 37,40 ****
--- 37,53 ----
          if nspam > 10 and nham > 10:
              db_status = "Database has %d good and %d spam." % (nham, nspam)
+             db_ratio = nham/float(nspam)
+             big = small = None
+             if db_ratio > 5.0:
+                 big = "ham"
+                 small = "spam"
+             elif db_ratio < (1/5.0):
+                 big = "spam"
+                 small = "ham"
+             if big is not None:                
+                 db_status = "%s\nWarning: you have much more %s than %s - " \
+                             "SpamBayes works best with approximately even " \
+                             "numbers of ham and spam." % (db_status, big,
+                                                           small)
          elif nspam > 0 or nham > 0:
              db_status = "Database only has %d good and %d spam - you should " \



From anadelonbrin at users.sourceforge.net  Tue Sep 16 20:03:56 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Tue Sep 16 20:03:59 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_server.py,1.4,1.5
Message-ID: 

Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv32667/scripts

Modified Files:
	sb_server.py 
Log Message:
Fix [ spambayes-Bugs-806632 ] sb_server failure when saving config

Index: sb_server.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_server.py,v
retrieving revision 1.4
retrieving revision 1.5
diff -C2 -d -r1.4 -r1.5
*** sb_server.py	13 Sep 2003 05:10:49 -0000	1.4
--- sb_server.py	17 Sep 2003 00:03:54 -0000	1.5
***************
*** 722,729 ****
      del proxyListeners[:]
  
!     # Close the database; we should anyway, and gdbm complains if we
!     # try to reopen it without closing it first.
!     state.bayes.store()
!     state.bayes.close()
  
      state = State()
--- 722,730 ----
      del proxyListeners[:]
  
!     # Close the database (if there is one); we should anyway, and gdbm
!     # complains if we try to reopen it without closing it first.
!     if hasattr(state, "bayes"):
!         state.bayes.store()
!         state.bayes.close()
  
      state = State()



From mhammond at users.sourceforge.net  Tue Sep 16 20:11:46 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Tue Sep 16 20:11:49 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs/resources
	dialogs.rc, 1.38, 1.39
Message-ID: 

Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs/resources
In directory sc8-pr-cvs1:/tmp/cvs-serv1872

Modified Files:
	dialogs.rc 
Log Message:
Give the database status room for a 3rd line.


Index: dialogs.rc
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/resources/dialogs.rc,v
retrieving revision 1.38
retrieving revision 1.39
diff -C2 -d -r1.38 -r1.39
*** dialogs.rc	8 Sep 2003 07:53:05 -0000	1.38
--- dialogs.rc	17 Sep 2003 00:11:44 -0000	1.39
***************
*** 490,504 ****
      LTEXT           "SpamBayes Version Here",IDC_VERSION,6,54,242,8
      LTEXT           "SpamBayes requires training before it is effective.  Click on the 'Training' tab, or use the Configuration Wizard to train.",
!                     IDC_STATIC,6,70,242,17
!     LTEXT           "Training database status:",IDC_STATIC,6,97,222,8
!     LTEXT           "123 spam messages; 456 good messages",
!                     IDC_TRAINING_STATUS,6,108,242,19,SS_SUNKEN
      CONTROL         "Enable SpamBayes",IDC_BUT_FILTER_ENABLE,"Button",
                      BS_AUTOCHECKBOX | WS_TABSTOP,6,173,97,11
      LTEXT           "Certain spam is moved to Folder1\nPossible spam is moved too",
!                     IDC_FILTER_STATUS,6,147,242,19,SS_SUNKEN
      PUSHBUTTON      "Reset Configuration...",IDC_BUT_RESET,6,190,84,15
      PUSHBUTTON      "Configuration Wizard...",IDC_BUT_WIZARD,164,190,84,15
!     LTEXT           "Filter status:",IDC_STATIC,6,136,222,8
      CONTROL         1062,IDC_LOGO_GRAPHIC,"Static",SS_BITMAP | 
                      SS_REALSIZEIMAGE,0,2,275,52
--- 490,504 ----
      LTEXT           "SpamBayes Version Here",IDC_VERSION,6,54,242,8
      LTEXT           "SpamBayes requires training before it is effective.  Click on the 'Training' tab, or use the Configuration Wizard to train.",
!                     IDC_STATIC,6,67,242,17
!     LTEXT           "Training database status:",IDC_STATIC,6,90,222,8
!     LTEXT           "123 spam messages; 456 good messages\r\nLine2\r\nLine3",
!                     IDC_TRAINING_STATUS,6,101,242,27,SS_SUNKEN
      CONTROL         "Enable SpamBayes",IDC_BUT_FILTER_ENABLE,"Button",
                      BS_AUTOCHECKBOX | WS_TABSTOP,6,173,97,11
      LTEXT           "Certain spam is moved to Folder1\nPossible spam is moved too",
!                     IDC_FILTER_STATUS,6,146,242,19,SS_SUNKEN
      PUSHBUTTON      "Reset Configuration...",IDC_BUT_RESET,6,190,84,15
      PUSHBUTTON      "Configuration Wizard...",IDC_BUT_WIZARD,164,190,84,15
!     LTEXT           "Filter status:",IDC_STATIC,6,135,222,8
      CONTROL         1062,IDC_LOGO_GRAPHIC,"Static",SS_BITMAP | 
                      SS_REALSIZEIMAGE,0,2,275,52



From anadelonbrin at users.sourceforge.net  Wed Sep 17 23:59:01 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Wed Sep 17 23:59:07 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_pop3dnd.py, 1.2,
	1.3 sb_server.py, 1.5, 1.6
Message-ID: 

Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv22711/scripts

Modified Files:
	sb_pop3dnd.py sb_server.py 
Log Message:
If we can't find a config file anywhere but the windows app data directory, then load it.

Swap various over to correct places in prep for 1.0a6.  If you were using notate_to, notate_subject, hammie:debug_header, or many of the pop3proxy storage options, you will need to update your config file.  These are in the most suitable places now.

Use urllib not urllib2 to shut down the proxy.  Why?  Because it works and the other doesn't.

Create the server strings for the ui *after* reading in the command line parameters.

Fix importing Options.py in ProxyUI.py for the binary.

Index: sb_pop3dnd.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_pop3dnd.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** sb_pop3dnd.py	6 Sep 2003 04:03:34 -0000	1.2
--- sb_pop3dnd.py	18 Sep 2003 03:58:59 -0000	1.3
***************
*** 863,872 ****
      app = Application("SpambayesIMAPServer")
  
!     spam_box = SpambayesMailbox("Spam", 0, options["imapserver",
!                                                    "spam_directory"])
!     unsure_box = SpambayesMailbox("Unsure", 1, options["imapserver",
!                                                        "unsure_directory"])
      ham_train_box = SpambayesMailbox("TrainAsHam", 2,
!                                      options["imapserver", "ham_directory"])
  
      spam_trainer = Trainer(spam_box, True)
--- 863,872 ----
      app = Application("SpambayesIMAPServer")
  
!     spam_box = SpambayesMailbox("Spam", 0, options["Storage",
!                                                    "spam_cache"])
!     unsure_box = SpambayesMailbox("Unsure", 1, options["Storage",
!                                                        "unknown_cache"])
      ham_train_box = SpambayesMailbox("TrainAsHam", 2,
!                                      options["Storage", "ham_cache"])
  
      spam_trainer = Trainer(spam_box, True)

Index: sb_server.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_server.py,v
retrieving revision 1.5
retrieving revision 1.6
diff -C2 -d -r1.5 -r1.6
*** sb_server.py	17 Sep 2003 00:03:54 -0000	1.5
--- sb_server.py	18 Sep 2003 03:58:59 -0000	1.6
***************
*** 461,469 ****
                      isSuppressedBulkHam = \
                          (cls == options["Headers", "header_ham_string"] and
!                          options["pop3proxy", "no_cache_bulk_ham"] and
                           msg.get('precedence') in ['bulk', 'list'])
  
                      # Suppress large messages if the options say so.
!                     size_limit = options["pop3proxy",
                                           "no_cache_large_messages"]
                      isTooBig = size_limit > 0 and \
--- 461,469 ----
                      isSuppressedBulkHam = \
                          (cls == options["Headers", "header_ham_string"] and
!                          options["Storage", "no_cache_bulk_ham"] and
                           msg.get('precedence') in ['bulk', 'list'])
  
                      # Suppress large messages if the options say so.
!                     size_limit = options["Storage",
                                           "no_cache_large_messages"]
                      isTooBig = size_limit > 0 and \
***************
*** 473,477 ****
                      # messages or suppressed bulk ham.
                      if (not state.isTest and
!                         options["pop3proxy", "cache_messages"] and
                          not isSuppressedBulkHam and not isTooBig):
                          # Write the message into the Unknown cache.
--- 473,477 ----
                      # messages or suppressed bulk ham.
                      if (not state.isTest and
!                         options["Storage", "cache_messages"] and
                          not isSuppressedBulkHam and not isTooBig):
                          # Write the message into the Unknown cache.
***************
*** 586,591 ****
          self.uiPort = options["html_ui", "port"]
          self.launchUI = options["html_ui", "launch_browser"]
!         self.gzipCache = options["pop3proxy", "cache_use_gzip"]
!         self.cacheExpiryDays = options["pop3proxy", "cache_expiry_days"]
          self.runTestServer = False
          self.isTest = False
--- 586,591 ----
          self.uiPort = options["html_ui", "port"]
          self.launchUI = options["html_ui", "launch_browser"]
!         self.gzipCache = options["Storage", "cache_use_gzip"]
!         self.cacheExpiryDays = options["Storage", "cache_expiry_days"]
          self.runTestServer = False
          self.isTest = False
***************
*** 634,657 ****
              # Create/open the Corpuses.  Use small cache sizes to avoid hogging
              # lots of memory.
!             map(ensureDir, [options["pop3proxy", "spam_cache"],
!                             options["pop3proxy", "ham_cache"],
!                             options["pop3proxy", "unknown_cache"]])
              if self.gzipCache:
                  factory = GzipFileMessageFactory()
              else:
                  factory = FileMessageFactory()
!             age = options["pop3proxy", "cache_expiry_days"]*24*60*60
              self.spamCorpus = ExpiryFileCorpus(age, factory,
!                                                options["pop3proxy",
                                                         "spam_cache"],
                                                 '[0123456789\-]*',
                                                 cacheSize=20)
              self.hamCorpus = ExpiryFileCorpus(age, factory,
!                                               options["pop3proxy",
                                                        "ham_cache"],
                                                '[0123456789\-]*',
                                                cacheSize=20)
              self.unknownCorpus = ExpiryFileCorpus(age, factory,
!                                             options["pop3proxy",
                                                      "unknown_cache"],
                                              '[0123456789\-]*',
--- 634,657 ----
              # Create/open the Corpuses.  Use small cache sizes to avoid hogging
              # lots of memory.
!             map(ensureDir, [options["Storage", "spam_cache"],
!                             options["Storage", "ham_cache"],
!                             options["Storage", "unknown_cache"]])
              if self.gzipCache:
                  factory = GzipFileMessageFactory()
              else:
                  factory = FileMessageFactory()
!             age = options["Storage", "cache_expiry_days"]*24*60*60
              self.spamCorpus = ExpiryFileCorpus(age, factory,
!                                                options["Storage",
                                                         "spam_cache"],
                                                 '[0123456789\-]*',
                                                 cacheSize=20)
              self.hamCorpus = ExpiryFileCorpus(age, factory,
!                                               options["Storage",
                                                        "ham_cache"],
                                                '[0123456789\-]*',
                                                cacheSize=20)
              self.unknownCorpus = ExpiryFileCorpus(age, factory,
!                                             options["Storage",
                                                      "unknown_cache"],
                                              '[0123456789\-]*',
***************
*** 766,771 ****
      # Shutdown as though through the web UI.  This will save the DB, allow
      # any open proxy connections to complete, etc.
!     from urllib2 import urlopen
!     from urllib import urlencode
      urlopen('http://localhost:%d/save' % state.uiPort,
              urlencode({'how': 'Save & shutdown'})).read()
--- 766,770 ----
      # Shutdown as though through the web UI.  This will save the DB, allow
      # any open proxy connections to complete, etc.
!     from urllib import urlopen, urlencode
      urlopen('http://localhost:%d/save' % state.uiPort,
              urlencode({'how': 'Save & shutdown'})).read()
***************
*** 810,815 ****
      print "and engine %s.\n" % (get_version_string(),)
  
-     prepare(state=state)
- 
      if 0 <= len(args) <= 2:
          # Normal usage, with optional server name and port number.
--- 809,812 ----
***************
*** 823,826 ****
--- 820,824 ----
              state.proxyPorts = [('', 110)]
  
+         prepare(state=state)
          start(state=state)
  



From anadelonbrin at users.sourceforge.net  Wed Sep 17 23:59:01 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Wed Sep 17 23:59:10 2003
Subject: [Spambayes-checkins] spambayes/spambayes ImapUI.py, 1.17,
	1.18 Options.py, 1.75, 1.76 ProxyUI.py, 1.21, 1.22 Version.py,
	1.22, 1.23 hammie.py, 1.10, 1.11 message.py, 1.36, 1.37
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv22711/spambayes

Modified Files:
	ImapUI.py Options.py ProxyUI.py Version.py hammie.py 
	message.py 
Log Message:
If we can't find a config file anywhere but the windows app data directory, then load it.

Swap various over to correct places in prep for 1.0a6.  If you were using notate_to, notate_subject, hammie:debug_header, or many of the pop3proxy storage options, you will need to update your config file.  These are in the most suitable places now.

Use urllib not urllib2 to shut down the proxy.  Why?  Because it works and the other doesn't.

Create the server strings for the ui *after* reading in the command line parameters.

Fix importing Options.py in ProxyUI.py for the binary.

Index: ImapUI.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/ImapUI.py,v
retrieving revision 1.17
retrieving revision 1.18
diff -C2 -d -r1.17 -r1.18
*** ImapUI.py	26 Aug 2003 04:30:41 -0000	1.17
--- ImapUI.py	18 Sep 2003 03:58:59 -0000	1.18
***************
*** 68,73 ****
      ('html_ui',               'allow_remote_connections'),
      ('Header Options',        None),
!     ('pop3proxy',             'notate_to'),
!     ('pop3proxy',             'notate_subject'),
      ('Storage Options',       None),
      ('Storage',               'persistent_storage_file'),
--- 68,73 ----
      ('html_ui',               'allow_remote_connections'),
      ('Header Options',        None),
!     ('Headers',             'notate_to'),
!     ('Headers',             'notate_subject'),
      ('Storage Options',       None),
      ('Storage',               'persistent_storage_file'),

Index: Options.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v
retrieving revision 1.75
retrieving revision 1.76
diff -C2 -d -r1.75 -r1.76
*** Options.py	16 Sep 2003 14:13:08 -0000	1.75
--- Options.py	18 Sep 2003 03:58:59 -0000	1.76
***************
*** 419,432 ****
  
    "Hammie": (
-     ("debug_header", "Add debug header", False,
-      """Enable debugging information in the header.""",
-      BOOLEAN, RESTORE),
- 
-     ("debug_header_name", "Debug header name", "X-Spambayes-Debug",
-      """Name of a debugging header for spambayes hackers, showing the
-      strongest clues that have resulted in the classification in the
-      standard header.""",
-      HEADER_NAME, RESTORE),
- 
      ("train_on_filter", "Train when filtering", False,
       """Train when filtering?  After filtering a message, hammie can then
--- 419,422 ----
***************
*** 470,473 ****
--- 460,517 ----
       working directory.""",
       FILE_WITH_PATH, DO_NOT_RESTORE),
+ 
+     ("cache_use_gzip", "Use gzip", False,
+      """Use gzip to compress the cache.""",
+      BOOLEAN, RESTORE),
+ 
+     ("cache_expiry_days", "Days before cached messages expire", 7,
+      """Messages will be expired from the cache after this many days.
+      After this time, you will no longer be able to train on these messages
+      (note this does not effect the copy of the message that you have in
+      your mail client).""",
+      INTEGER, RESTORE),
+ 
+     ("spam_cache", "Spam cache directory", "pop3proxy-spam-cache",
+      """Directory that SpamBayes should cache spam in.  If this does
+      not exist, it will be created.""",
+      PATH, DO_NOT_RESTORE),
+ 
+     ("ham_cache", "Ham cache directory", "pop3proxy-ham-cache",
+      """Directory that SpamBayes should cache ham in.  If this does
+      not exist, it will be created.""",
+      PATH, DO_NOT_RESTORE),
+ 
+     ("unknown_cache", "Unknown cache directory", "pop3proxy-unknown-cache",
+      """Directory that SpamBayes should cache unclassified messages in.
+      If this does not exist, it will be created.""",
+      PATH, DO_NOT_RESTORE),
+ 
+     ("cache_messages", "Cache messages", True,
+      """You can disable the pop3proxy caching of messages.  This
+      will make the proxy a bit faster, and make it use less space
+      on your hard drive.  The proxy uses its cache for reviewing
+      and training of messages, so if you disable caching you won't
+      be able to do further training unless you re-enable it.
+      Thus, you should only turn caching off when you are satisfied
+      with the filtering that Spambayes is doing for you.""",
+      BOOLEAN, RESTORE),
+ 
+     ("no_cache_bulk_ham", "Suppress caching of bulk ham", False,
+      """Where message caching is enabled, this option suppresses caching
+      of messages which are classified as ham and marked as
+      'Precedence: bulk' or 'Precedence: list'.  If you subscribe to a
+      high-volume mailing list then your 'Review messages' page can be
+      overwhelmed with list messages, making training a pain.  Once you've
+      trained Spambayes on enough list traffic, you can use this option
+      to prevent that traffic showing up in 'Review messages'.""",
+      BOOLEAN, RESTORE),
+ 
+     ("no_cache_large_messages", "Maximum size of cached messages", 0,
+      """Where message caching is enabled, this option suppresses caching
+      of messages which are larger than this value (measured in bytes).
+      If you receive a lot of messages that include large attachments
+      (and are correctly classified), you may not wish to cache these.
+      If you set this to zero (0), then this option will have no effect.""",
+      INTEGER, RESTORE),
    ),
  
***************
*** 588,591 ****
--- 632,656 ----
       option adds this information to a header added to each message.""",
       BOOLEAN, RESTORE),
+ 
+     ("notate_to", "Notate to", (),
+      """Some email clients (Outlook Express, for example) can only set up
+      filtering rules on a limited set of headers.  These clients cannot
+      test for the existence/value of an arbitrary header and filter mail
+      based on that information.  To accommodate these kind of mail clients,
+      you can add "spam", "ham", or "unsure" to the recipient list.  A
+      filter rule can then use this to see if one of these words (followed
+      by a comma) is in the recipient list, and route the mail to an
+      appropriate folder, or take whatever other action is supported and
+      appropriate for the mail classification.
+ 
+      As it interferes with replying, you may only wish to do this for
+      spam messages; simply tick the boxes of the classifications take
+      should be identified in this fashion.""",
+      ("ham", "spam", "unsure"), RESTORE),
+ 
+     ("notate_subject", "Classify in subject: header", (),
+      """This option will add the same information as 'Notate To',
+      but to the start of the mail subject line.""",
+      ("ham", "spam", "unsure"), RESTORE),
    ),
  
***************
*** 617,699 ****
       specify the same number of ports as servers, separated by commas.""",
       SERVER, DO_NOT_RESTORE),
- 
-     ("cache_use_gzip", "Use gzip", False,
-      """Use gzip to compress the cache.""",
-      BOOLEAN, RESTORE),
- 
-     ("cache_expiry_days", "Days before cached messages expire", 7,
-      """Messages will be expired from the cache after this many days.
-      After this time, you will no longer be able to train on these messages
-      (note this does not effect the copy of the message that you have in
-      your mail client).""",
-      INTEGER, RESTORE),
- 
-     ("spam_cache", "Spam cache directory", "pop3proxy-spam-cache",
-      """Directory that SpamBayes should cache spam in.  If this does
-      not exist, it will be created.""",
-      PATH, DO_NOT_RESTORE),
- 
-     ("ham_cache", "Ham cache directory", "pop3proxy-ham-cache",
-      """Directory that SpamBayes should cache ham in.  If this does
-      not exist, it will be created.""",
-      PATH, DO_NOT_RESTORE),
- 
-     ("unknown_cache", "Unknown cache directory", "pop3proxy-unknown-cache",
-      """Directory that SpamBayes should cache unclassified messages in.
-      If this does not exist, it will be created.""",
-      PATH, DO_NOT_RESTORE),
- 
-     ("notate_to", "Notate to", (),
-      """Some email clients (Outlook Express, for example) can only set up
-      filtering rules on a limited set of headers.  These clients cannot
-      test for the existence/value of an arbitrary header and filter mail
-      based on that information.  To accommodate these kind of mail clients,
-      you can add "spam", "ham", or "unsure" to the recipient list.  A
-      filter rule can then use this to see if one of these words (followed
-      by a comma) is in the recipient list, and route the mail to an
-      appropriate folder, or take whatever other action is supported and
-      appropriate for the mail classification.
- 
-      As it interferes with replying, you may only wish to do this for
-      spam messages; simply tick the boxes of the classifications take
-      should be identified in this fashion.""",
-      ("ham", "spam", "unsure"), RESTORE),
- 
-     ("notate_subject", "Classify in subject: header", (),
-      """This option will add the same information as 'Notate To',
-      but to the start of the mail subject line.""",
-      ("ham", "spam", "unsure"), RESTORE),
- 
-     ("cache_messages", "Cache messages", True,
-      """You can disable the pop3proxy caching of messages.  This
-      will make the proxy a bit faster, and make it use less space
-      on your hard drive.  The proxy uses its cache for reviewing
-      and training of messages, so if you disable caching you won't
-      be able to do further training unless you re-enable it.
-      Thus, you should only turn caching off when you are satisfied
-      with the filtering that Spambayes is doing for you.""",
-      BOOLEAN, RESTORE),
- 
-     ("no_cache_bulk_ham", "Suppress caching of bulk ham", False,
-      """Where message caching is enabled, this option suppresses caching
-      of messages which are classified as ham and marked as
-      'Precedence: bulk' or 'Precedence: list'.  If you subscribe to a
-      high-volume mailing list then your 'Review messages' page can be
-      overwhelmed with list messages, making training a pain.  Once you've
-      trained Spambayes on enough list traffic, you can use this option
-      to prevent that traffic showing up in 'Review messages'.""",
-      BOOLEAN, RESTORE),
- 
-     ("no_cache_large_messages", "Maximum size of cached messages", 0,
-      """Where message caching is enabled, this option suppresses caching
-      of messages which are larger than this value (measured in bytes).
-      If you receive a lot of messages that include large attachments
-      (and are correctly classified), you may not wish to cache these.
-      If you set this to zero (0), then this option will have no effect.""",
-      INTEGER, RESTORE),
    ),
  
    "smtpproxy" : (
- 
      ("remote_servers", "Remote Servers", (),
       """The Spambayes SMTP proxy intercepts outgoing email - if you
--- 682,688 ----
***************
*** 903,918 ****
       """The port to serve the SpamBayes IMAP server on.""",
       PORT, RESTORE),
- 
-     ("spam_directory", "Spam directory", "imapserver-spam",
-      """The directory to store spam messages in.""",
-      PATH, DO_NOT_RESTORE),
-     
-     ("ham_directory", "Ham directory", "imapserver-ham",
-      """The directory to store ham messages in.""",
-      PATH, DO_NOT_RESTORE),
-     
-     ("unsure_directory", "Unsure directory", "imapserver-unsure",
-      """The directory to store unsure messages in.""",
-      PATH, DO_NOT_RESTORE),
    ),
   
--- 892,895 ----
***************
*** 991,994 ****
--- 968,983 ----
                  # Not everyone is unicode aware - keep it a string.
                  optionsPathname = optionsPathname.encode("mbcs")
+                 # If the file exists, then load it.
+                 if os.path.exists(optionsPathname):
+                     options.merge_file(optionsPathname)
+                 else:
+                     # If the file doesn't exist, then let's get the user to
+                     # store their databases here as well, by default, and
+                     # save the file.
+                     db_name = os.path.join(windowsUserDirectory,
+                                            "statistics_database.db").\
+                                            encode("mbcs")
+                     options["Storage", "persistent_storage_file"] = db_name
+                     options.update_file(optionsPathname)
              except os.error:
                  # unable to make the directory - stick to default.

Index: ProxyUI.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/ProxyUI.py,v
retrieving revision 1.21
retrieving revision 1.22
diff -C2 -d -r1.21 -r1.22
*** ProxyUI.py	9 Sep 2003 06:05:15 -0000	1.21
--- ProxyUI.py	18 Sep 2003 03:58:59 -0000	1.22
***************
*** 68,72 ****
  import tokenizer
  import UserInterface
! from Options import options
  import spambayes.mboxutils
  from email.Iterators import typed_subpart_iterator
--- 68,72 ----
  import tokenizer
  import UserInterface
! from spambayes.Options import options
  import spambayes.mboxutils
  from email.Iterators import typed_subpart_iterator
***************
*** 89,94 ****
      ('html_ui',             'http_password'),
      ('Header Options',      None),
!     ('pop3proxy',           'notate_to'),
!     ('pop3proxy',           'notate_subject'),
      ('SMTP Proxy Options',  None),
      ('smtpproxy',           'remote_servers'),
--- 89,94 ----
      ('html_ui',             'http_password'),
      ('Header Options',      None),
!     ('Headers',             'notate_to'),
!     ('Headers',             'notate_subject'),
      ('SMTP Proxy Options',  None),
      ('smtpproxy',           'remote_servers'),
***************
*** 100,106 ****
      ('Storage',             'persistent_storage_file'),
      ('Storage',             'messageinfo_storage_file'),
!     ('pop3proxy',           'cache_messages'),
!     ('pop3proxy',           'no_cache_bulk_ham'),
!     ('pop3proxy',           'no_cache_large_messages'),
      ('Statistics Options',  None),
      ('Categorization',      'ham_cutoff'),
--- 100,106 ----
      ('Storage',             'persistent_storage_file'),
      ('Storage',             'messageinfo_storage_file'),
!     ('Storage',             'cache_messages'),
!     ('Storage',             'no_cache_bulk_ham'),
!     ('Storage',             'no_cache_large_messages'),
      ('Statistics Options',  None),
      ('Categorization',      'ham_cutoff'),
***************
*** 126,134 ****
      ('Storage Options',     None),
      ('Storage',             'persistent_use_database'),
!     ('pop3proxy',           'cache_expiry_days'),
!     ('pop3proxy',           'cache_use_gzip'),
!     ('pop3proxy',           'ham_cache'),
!     ('pop3proxy',           'spam_cache'),
!     ('pop3proxy',           'unknown_cache'),
      ('Tokenising Options',  None),
      ('Tokenizer',           'extract_dow'),
--- 126,134 ----
      ('Storage Options',     None),
      ('Storage',             'persistent_use_database'),
!     ('Storage',             'cache_expiry_days'),
!     ('Storage',             'cache_use_gzip'),
!     ('Storage',             'ham_cache'),
!     ('Storage',             'spam_cache'),
!     ('Storage',             'unknown_cache'),
      ('Tokenising Options',  None),
      ('Tokenizer',           'extract_dow'),

Index: Version.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Version.py,v
retrieving revision 1.22
retrieving revision 1.23
diff -C2 -d -r1.22 -r1.23
*** Version.py	15 Sep 2003 01:00:18 -0000	1.22
--- Version.py	18 Sep 2003 03:58:59 -0000	1.23
***************
*** 90,94 ****
              "Date":             "September 2003",
              "InterfaceVersion":             0.02,
!             "InterfaceDescription":        "SpamBayes IMAP Filter Web Interface Alpha2",
              "Full Description": """%(Description)s, version %(Version)s (%(Date)s),
  using %(InterfaceDescription)s, version %(InterfaceVersion)s""",
--- 90,94 ----
              "Date":             "September 2003",
              "InterfaceVersion":             0.02,
!             "InterfaceDescription":        "SpamBayes IMAP Server Web Interface Alpha2",
              "Full Description": """%(Description)s, version %(Version)s (%(Date)s),
  using %(InterfaceDescription)s, version %(InterfaceVersion)s""",

Index: hammie.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/hammie.py,v
retrieving revision 1.10
retrieving revision 1.11
diff -C2 -d -r1.10 -r1.11
*** hammie.py	10 Sep 2003 04:12:20 -0000	1.10
--- hammie.py	18 Sep 2003 03:58:59 -0000	1.11
***************
*** 85,89 ****
  
          Returns the same message with a new disposition header.
- 
          """
  
--- 85,88 ----
***************
*** 95,101 ****
              ham_cutoff = options["Categorization", "ham_cutoff"]
          if debugheader == None:
!             debugheader = options["Hammie", "debug_header_name"]
          if debug == None:
!             debug = options["Hammie", "debug_header"]
          if train == None:
              train = options["Hammie", "train_on_filter"]
--- 94,100 ----
              ham_cutoff = options["Categorization", "ham_cutoff"]
          if debugheader == None:
!             debugheader = options["Headers", "evidence_header_name"]
          if debug == None:
!             debug = options["Headers", "include_evidence"]
          if train == None:
              train = options["Hammie", "train_on_filter"]

Index: message.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/message.py,v
retrieving revision 1.36
retrieving revision 1.37
diff -C2 -d -r1.36 -r1.37
*** message.py	8 Sep 2003 14:46:57 -0000	1.36
--- message.py	18 Sep 2003 03:58:59 -0000	1.37
***************
*** 321,325 ****
          # allow filtering in 'stripped down' mailers like Outlook Express,
          # so for the moment, they stay in.
!         if disposition in options["pop3proxy", "notate_to"]:
              try:
                  self.replace_header("To", "%s,%s" % (disposition,
--- 321,325 ----
          # allow filtering in 'stripped down' mailers like Outlook Express,
          # so for the moment, they stay in.
!         if disposition in options["Headers", "notate_to"]:
              try:
                  self.replace_header("To", "%s,%s" % (disposition,
***************
*** 328,332 ****
                  self["To"] = disposition
  
!         if disposition in options["pop3proxy", "notate_subject"]:
              try:
                  self.replace_header("Subject", "%s,%s" % (disposition,
--- 328,332 ----
                  self["To"] = disposition
  
!         if disposition in options["Headers", "notate_subject"]:
              try:
                  self.replace_header("Subject", "%s,%s" % (disposition,



From anadelonbrin at users.sourceforge.net  Thu Sep 18 00:12:53 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 18 00:12:58 2003
Subject: [Spambayes-checkins] spambayes/spambayes Options.py,1.76,1.77
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv24992/spambayes

Modified Files:
	Options.py 
Log Message:
Store the user's caches and messageinfo.db file in the app data folder as well (by default),
if we have everything else there.

Index: Options.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v
retrieving revision 1.76
retrieving revision 1.77
diff -C2 -d -r1.76 -r1.77
*** Options.py	18 Sep 2003 03:58:59 -0000	1.76
--- Options.py	18 Sep 2003 04:12:51 -0000	1.77
***************
*** 973,982 ****
                  else:
                      # If the file doesn't exist, then let's get the user to
!                     # store their databases here as well, by default, and
!                     # save the file.
                      db_name = os.path.join(windowsUserDirectory,
!                                            "statistics_database.db").\
!                                            encode("mbcs")
!                     options["Storage", "persistent_storage_file"] = db_name
                      options.update_file(optionsPathname)
              except os.error:
--- 973,995 ----
                  else:
                      # If the file doesn't exist, then let's get the user to
!                     # store their databases and caches here as well, by
!                     # default, and save the file.
                      db_name = os.path.join(windowsUserDirectory,
!                                            "statistics_database.db")
!                     mi_db = os.path.join(windowsUserDirectory,
!                                            "message_info_database.db")
!                     h_cache = os.path.join(windowsUserDirectory,
!                                            "ham_cache").encode("mbcs")
!                     u_cache = os.path.join(windowsUserDirectory,
!                                            "unknown_cache").encode("mbcs")
!                     s_cache = os.path.join(windowsUserDirectory,
!                                            "spam_cache").encode("mbcs")
!                     options["Storage", "spam_cache"] = s_cache
!                     options["Storage", "ham_cache"] = h_cache
!                     options["Storage", "unknown_cache"] = u_cache
!                     options["Storage", "persistent_storage_file"] = \
!                                        db_name.encode("mbcs")
!                     options["Storage", "messageinfo_storage_file"] = \
!                                        mi_db.encode("mbcs")
                      options.update_file(optionsPathname)
              except os.error:



From mhammond at users.sourceforge.net  Thu Sep 18 00:15:27 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Thu Sep 18 00:15:33 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_service.py, 1.13,
	1.14
Message-ID: 

Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv25317

Modified Files:
	pop3proxy_service.py 
Log Message:
There is no good reason not to use win32api.GetUserName()


Index: pop3proxy_service.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_service.py,v
retrieving revision 1.13
retrieving revision 1.14
diff -C2 -d -r1.13 -r1.14
*** pop3proxy_service.py	16 Sep 2003 14:14:11 -0000	1.13
--- pop3proxy_service.py	18 Sep 2003 04:15:24 -0000	1.14
***************
*** 117,121 ****
          from spambayes.Options import optionsPathname
          extra = " as user '%s', using config file '%s'" \
!                 % (win32api.GetUserNameEx(2).encode("mbcs"),
                     optionsPathname)
          import servicemanager
--- 117,121 ----
          from spambayes.Options import optionsPathname
          extra = " as user '%s', using config file '%s'" \
!                 % (win32api.GetUserName(),
                     optionsPathname)
          import servicemanager



From anadelonbrin at users.sourceforge.net  Thu Sep 18 00:31:01 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 18 00:31:04 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000
	default_bayes_customize.ini, 1.8, 1.9
Message-ID: 

Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv27180/Outlook2000

Modified Files:
	default_bayes_customize.ini 
Log Message:
Change the Outlook plug-in to use the general default (currently False) for the
experimental_ham_spam_imbalance adjustment.

Change the default for address_headers to include to, cc, reply-to, and sender as per
Tim's suggestion.

Index: default_bayes_customize.ini
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/default_bayes_customize.ini,v
retrieving revision 1.8
retrieving revision 1.9
diff -C2 -d -r1.8 -r1.9
*** default_bayes_customize.ini	15 Sep 2003 01:08:11 -0000	1.8
--- default_bayes_customize.ini	18 Sep 2003 04:30:59 -0000	1.9
***************
*** 16,26 ****
  record_header_absence: True
  
- # These should help.  All but "from" are disabled by default, because
- # they're killer-good clues for bad reasons when using mixed-source
- # data.
- address_headers: from to cc sender reply-to
- 
- 
  [Classifier]
  # This will probably go away if testing confirms it's a Good Thing.
! experimental_ham_spam_imbalance_adjustment: True
--- 16,22 ----
  record_header_absence: True
  
  [Classifier]
  # This will probably go away if testing confirms it's a Good Thing.
! # Testing seems to be indicating that this is a Bad Thing, so we
! # leave it disabled.
! #experimental_ham_spam_imbalance_adjustment: True



From anadelonbrin at users.sourceforge.net  Thu Sep 18 00:31:01 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 18 00:31:06 2003
Subject: [Spambayes-checkins] spambayes/spambayes Options.py,1.77,1.78
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv27180/spambayes

Modified Files:
	Options.py 
Log Message:
Change the Outlook plug-in to use the general default (currently False) for the
experimental_ham_spam_imbalance adjustment.

Change the default for address_headers to include to, cc, reply-to, and sender as per
Tim's suggestion.

Index: Options.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v
retrieving revision 1.77
retrieving revision 1.78
diff -C2 -d -r1.77 -r1.78
*** Options.py	18 Sep 2003 04:12:51 -0000	1.77
--- Options.py	18 Sep 2003 04:30:59 -0000	1.78
***************
*** 101,105 ****
       BOOLEAN, RESTORE),
  
!     ("address_headers", "Address headers to mine", ("from",),
       """Mine the following address headers. If you have mixed source
       corpuses (as opposed to a mixed sauce walrus, which is delicious!)
--- 101,105 ----
       BOOLEAN, RESTORE),
  
!     ("address_headers", "Address headers to mine", ("from", "to", "cc", "sender", "reply-to"),
       """Mine the following address headers. If you have mixed source
       corpuses (as opposed to a mixed sauce walrus, which is delicious!)



From mhammond at users.sourceforge.net  Thu Sep 18 01:36:41 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Thu Sep 18 01:36:45 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000 about.html,1.23,1.24
Message-ID: 

Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv3161

Modified Files:
	about.html 
Log Message:
Remove an accidental 
         tag.


Index: about.html
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/about.html,v
retrieving revision 1.23
retrieving revision 1.24
diff -C2 -d -r1.23 -r1.24
*** about.html	10 Sep 2003 07:33:39 -0000	1.23
--- about.html	18 Sep 2003 05:36:39 -0000	1.24
***************
*** 41,45 ****
  When Outlook is under load, SpamBayes may occasionally miss some
  messages (as Outlook doesn't tell us they appeared).  Also, if you
! use builtin Outlook rules, you may find that occasionally they also
        
  fail to filter messages (as both SpamBayes and the builtin rules
  "fight" over processing the message).  Both of these
--- 41,45 ----
  When Outlook is under load, SpamBayes may occasionally miss some
  messages (as Outlook doesn't tell us they appeared).  Also, if you
! use builtin Outlook rules, you may find that occasionally they also
  fail to filter messages (as both SpamBayes and the builtin rules
  "fight" over processing the message).  Both of these



From montanaro at users.sourceforge.net  Thu Sep 18 09:55:14 2003
From: montanaro at users.sourceforge.net (Skip Montanaro)
Date: Thu Sep 18 09:55:22 2003
Subject: [Spambayes-checkins] spambayes/spambayes Options.py, 1.78,
	1.79 ProxyUI.py, 1.22, 1.23
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv27691/spambayes

Modified Files:
	Options.py ProxyUI.py 
Log Message:
Place a threshold on the number of items displayed per section.  The default
is 10,000, large enough so most people shouldn't notice the change.  If you
have a large amount of untrained mail, setting a smallish (100 or so) value
for the "rows_per_section" variable in the "html_ui" section of your options
file gives you a reasonable amount of mail to consider.  (I personally go
numb after scanning a couple dozen subjects.)  It also means that if you
click one of the section headers to set all the radio buttons to a new value
a slow JavaScript implementation won't have you waiting for the next
Spammish Inquisition to arrive.


Index: Options.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v
retrieving revision 1.78
retrieving revision 1.79
diff -C2 -d -r1.78 -r1.79
*** Options.py	18 Sep 2003 04:30:59 -0000	1.78
--- Options.py	18 Sep 2003 13:55:11 -0000	1.79
***************
*** 785,788 ****
--- 785,793 ----
       authorized user password here.""",
       r"[\w]+", RESTORE),
+ 
+     ("rows_per_section", "Rows per section", 10000,
+      """Number of rows to display per ham/spam/unsure section.""",
+      INTEGER, RESTORE),
+ 
    ),
  

Index: ProxyUI.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/ProxyUI.py,v
retrieving revision 1.22
retrieving revision 1.23
diff -C2 -d -r1.22 -r1.23
*** ProxyUI.py	18 Sep 2003 03:58:59 -0000	1.22
--- ProxyUI.py	18 Sep 2003 13:55:11 -0000	1.23
***************
*** 249,253 ****
          if not options["html_ui", "display_to"]:
              del table.to_header
!         for key, messageInfo in keyedMessageInfo:
              row = self.html.reviewRow.clone()
              if label == 'Spam':
--- 249,254 ----
          if not options["html_ui", "display_to"]:
              del table.to_header
!         nrows = options["html_ui", "rows_per_section"]
!         for key, messageInfo in keyedMessageInfo[:nrows]:
              row = self.html.reviewRow.clone()
              if label == 'Spam':



From montanaro at users.sourceforge.net  Thu Sep 18 10:04:33 2003
From: montanaro at users.sourceforge.net (Skip Montanaro)
Date: Thu Sep 18 10:04:54 2003
Subject: [Spambayes-checkins] spambayes/spambayes storage.py,1.33,1.34
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv30688

Modified Files:
	storage.py 
Log Message:
* one import per line
* it's not necessary to import sys at the bottom


Index: storage.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v
retrieving revision 1.33
retrieving revision 1.34
diff -C2 -d -r1.33 -r1.34
*** storage.py	11 Sep 2003 08:34:42 -0000	1.33
--- storage.py	18 Sep 2003 14:04:31 -0000	1.34
***************
*** 63,67 ****
          return not not val
  
! import sys, types
  from spambayes import classifier
  from spambayes.Options import options
--- 63,68 ----
          return not not val
  
! import sys
! import types
  from spambayes import classifier
  from spambayes.Options import options
***************
*** 678,681 ****
  
  if __name__ == '__main__':
-     import sys
      print >> sys.stderr, __doc__
--- 679,681 ----



From montanaro at users.sourceforge.net  Thu Sep 18 11:36:32 2003
From: montanaro at users.sourceforge.net (Skip Montanaro)
Date: Thu Sep 18 11:36:35 2003
Subject: [Spambayes-checkins] spambayes/spambayes hammiebulk.py,1.9,1.10
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv17627/spambayes

Modified Files:
	hammiebulk.py 
Log Message:
Minor performance boost when training on lots of mail.  Only display every
tenth message number.  More time is spent training and less time waiting
for i/o.


Index: hammiebulk.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/hammiebulk.py,v
retrieving revision 1.9
retrieving revision 1.10
diff -C2 -d -r1.9 -r1.10
*** hammiebulk.py	13 Aug 2003 22:48:38 -0000	1.9
--- hammiebulk.py	18 Sep 2003 15:36:30 -0000	1.10
***************
*** 79,85 ****
      for msg in mbox:
          i += 1
!         sys.stdout.write("\r%6d" % i)
!         sys.stdout.flush()
          h.train(msg, is_spam)
      print
  
--- 79,88 ----
      for msg in mbox:
          i += 1
!         if i % 10 == 0:
!             sys.stdout.write("\r%6d" % i)
!             sys.stdout.flush()
          h.train(msg, is_spam)
+     sys.stdout.write("\r%6d" % i)
+     sys.stdout.flush()
      print
  
***************
*** 90,96 ****
      for msg in mbox:
          i += 1
!         sys.stdout.write("\r%6d" % i)
!         sys.stdout.flush()
          h.untrain(msg, is_spam)
      print
  
--- 93,102 ----
      for msg in mbox:
          i += 1
!         if i % 10 == 0:
!             sys.stdout.write("\r%6d" % i)
!             sys.stdout.flush()
          h.untrain(msg, is_spam)
+     sys.stdout.write("\r%6d" % i)
+     sys.stdout.flush()
      print
  



From montanaro at users.sourceforge.net  Thu Sep 18 11:36:32 2003
From: montanaro at users.sourceforge.net (Skip Montanaro)
Date: Thu Sep 18 11:36:37 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_mboxtrain.py,1.2,1.3
Message-ID: 

Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv17627/scripts

Modified Files:
	sb_mboxtrain.py 
Log Message:
Minor performance boost when training on lots of mail.  Only display every
tenth message number.  More time is spent training and less time waiting
for i/o.


Index: sb_mboxtrain.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_mboxtrain.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** sb_mboxtrain.py	9 Sep 2003 07:03:54 -0000	1.2
--- sb_mboxtrain.py	18 Sep 2003 15:36:30 -0000	1.3
***************
*** 107,112 ****
              continue
          counter += 1
!         if loud:
!             sys.stdout.write("  %s        \r" % fn)
              sys.stdout.flush()
          f = file(cfn, "rb")
--- 107,112 ----
              continue
          counter += 1
!         if loud and counter % 10 == 0:
!             sys.stdout.write("\r%6d" % counter)
              sys.stdout.flush()
          f = file(cfn, "rb")
***************
*** 128,133 ****
  
      if loud:
!         print ("  Trained %d out of %d messages                " %
!                (trained, counter))
  
  def mbox_train(h, path, is_spam, force):
--- 128,134 ----
  
      if loud:
!         sys.stdout.write("\r%6d" % counter)
!         sys.stdout.write("\r  Trained %d out of %d messages\n" %
!                          (trained, counter))
  
  def mbox_train(h, path, is_spam, force):
***************
*** 152,157 ****
      for msg in mbox:
          counter += 1
!         if loud:
!             sys.stdout.write("  %s\r" % counter)
              sys.stdout.flush()
          if msg_train(h, msg, is_spam, force):
--- 153,158 ----
      for msg in mbox:
          counter += 1
!         if loud and counter % 10 == 0:
!             sys.stdout.write("\r%6d" % counter)
              sys.stdout.flush()
          if msg_train(h, msg, is_spam, force):
***************
*** 181,186 ****
      f.close()
      if loud:
!         print ("  Trained %d out of %d messages                " %
!                (trained, counter))
  
  def mhdir_train(h, path, is_spam, force):
--- 182,188 ----
      f.close()
      if loud:
!         sys.stdout.write("\r%6d" % counter)
!         sys.stdout.write("\r  Trained %d out of %d messages\n" %
!                          (trained, counter))
  
  def mhdir_train(h, path, is_spam, force):
***************
*** 199,204 ****
          cfn = fn
          tfn = os.path.join(path, "spambayes.tmp")
!         if loud:
!             sys.stdout.write("  %s        \r" % fn)
              sys.stdout.flush()
          f = file(fn, "rb")
--- 201,206 ----
          cfn = fn
          tfn = os.path.join(path, "spambayes.tmp")
!         if loud and counter % 10 == 0:
!             sys.stdout.write("\r%6d" % counter)
              sys.stdout.flush()
          f = file(fn, "rb")
***************
*** 218,223 ****
  
      if loud:
!         print ("  Trained %d out of %d messages                " %
!                (trained, counter))
  
  def train(h, path, is_spam, force, trainnew, removetrained):
--- 220,226 ----
  
      if loud:
!         sys.stdout.write("\r%6d" % counter)
!         sys.stdout.write("\r  Trained %d out of %d messages\n" %
!                          (trained, counter))
  
  def train(h, path, is_spam, force, trainnew, removetrained):



From richie at entrian.com  Thu Sep 18 14:17:20 2003
From: richie at entrian.com (Richie Hindle)
Date: Thu Sep 18 15:46:07 2003
Subject: [Spambayes-checkins] spambayes/spambayes storage.py,1.33,1.34
In-Reply-To: 
References: 
Message-ID: 


[Skip]
> * one import per line

I've never understood why the coding standard recommends that - is there a
concrete reason I'm missing, or is it a matter of taste?

-- 
Richie Hindle
richie@entrian.com


From montanaro at users.sourceforge.net  Thu Sep 18 17:00:13 2003
From: montanaro at users.sourceforge.net (Skip Montanaro)
Date: Thu Sep 18 17:00:15 2003
Subject: [Spambayes-checkins] spambayes/spambayes tokenizer.py,1.14,1.15
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv27208

Modified Files:
	tokenizer.py 
Log Message:
worm around possible email pkg bug - see code comment for msg thread


Index: tokenizer.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/tokenizer.py,v
retrieving revision 1.14
retrieving revision 1.15
diff -C2 -d -r1.14 -r1.15
*** tokenizer.py	5 Sep 2003 01:15:28 -0000	1.14
--- tokenizer.py	18 Sep 2003 21:00:10 -0000	1.15
***************
*** 805,813 ****
          yield 'content-disposition:' + x.lower()
  
!     fname = msg.get_filename()
!     if fname is not None:
!         for x in crack_filename(fname):
!             yield 'filename:' + x
! 
      if 0:   # disabled; see comment before function
          x = msg.get('content-transfer-encoding')
--- 805,820 ----
          yield 'content-disposition:' + x.lower()
  
!     try:
!         fname = msg.get_filename()
!         if fname is not None:
!             for x in crack_filename(fname):
!                 yield 'filename:' + x
!     except TypeError:
!         # bug in email pkg?  see the thread beginning at
!         # http://mail.python.org/pipermail/spambayes/2003-September/008006.html
!         # and
!         # http://mail.python.org/pipermail/spambayes-dev/2003-September/001177.html
!         yield "filename:"
!         
      if 0:   # disabled; see comment before function
          x = msg.get('content-transfer-encoding')



From anadelonbrin at users.sourceforge.net  Thu Sep 18 18:06:02 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 18 18:06:06 2003
Subject: [Spambayes-checkins] spambayes MANIFEST.in,1.6,1.7
Message-ID: 

Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv8875

Modified Files:
	MANIFEST.in 
Log Message:
Add extra files needed for pop3proxy_tray.

Index: MANIFEST.in
===================================================================
RCS file: /cvsroot/spambayes/spambayes/MANIFEST.in,v
retrieving revision 1.6
retrieving revision 1.7
diff -C2 -d -r1.6 -r1.7
*** MANIFEST.in	6 Sep 2003 03:12:46 -0000	1.6
--- MANIFEST.in	18 Sep 2003 22:05:59 -0000	1.7
***************
*** 6,10 ****
  recursive-include utilities *.py *.txt
  recursive-include testtools *.py *.txt
! recursive-include windows *.py *.txt
  include *.txt *.py
  
--- 6,10 ----
  recursive-include utilities *.py *.txt
  recursive-include testtools *.py *.txt
! recursive-include windows *.py *.txt *.h *.rc *.ico
  include *.txt *.py
  



From anadelonbrin at users.sourceforge.net  Thu Sep 18 18:20:39 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 18 18:20:42 2003
Subject: [Spambayes-checkins] spambayes CHANGELOG.txt,1.18,1.19
Message-ID: 

Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv12399

Modified Files:
	CHANGELOG.txt 
Log Message:
Bring up to date.

Index: CHANGELOG.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/CHANGELOG.txt,v
retrieving revision 1.18
retrieving revision 1.19
diff -C2 -d -r1.18 -r1.19
*** CHANGELOG.txt	9 Sep 2003 08:21:19 -0000	1.18
--- CHANGELOG.txt	18 Sep 2003 22:20:36 -0000	1.19
***************
*** 3,6 ****
--- 3,41 ----
  Alpha Release 6
  ===============
+ Skip Montanaro    19/09/2003  Worm around a possible email pkg bug.
+ Skip Montanaro    19/09/2003  hammiebulk & mboxtrain: Minor performance boost when training on lots of mail.
+ Skip Montanaro    19/09/2003  Place a threshold on the number of items displayed per section in the review page of the ui.
+ Tony Meyer        18/09/2003  Change the Outlook plug-in to use the general default (currently False) for the experimental_ham_spam_imbalance adjustment.
+ Tony Meyer        18/09/2003  Change the default for address_headers to include to, cc, reply-to, and sender as per Tim's suggestion.
+ Tony Meyer        18/09/2003  Store the user's caches and messageinfo.db file in the app data folder as well (by default), if we have everything else there.
+ Tony Meyer        18/09/2003  If we can't find a config file anywhere but the windows app data directory, then load it.
+ Tony Meyer        18/09/2003  Use urllib not urllib2 to shut down the proxy.
+ Tony Meyer        18/09/2003  Create the server strings for the ui *after* reading in the command line parameters.
+ Tony Meyer        18/09/2003  Move notate_to and notate_subject options to the "Headers" section.
+ Tony Meyer        18/09/2003  Move all the storage options in the "pop3proxy" section to the "Storage" section.
+ Tony Meyer        18/09/2003  Remove the hammie debug options, and use the "Headers" evidence options instead.
+ Tony Meyer        17/09/2003  Fix [ spambayes-Bugs-806632 ] sb_server failure when saving config
+ Tony Meyer        17/09/2003  Outlook: Add a warning for those with highly (>5 times) imbalanced ham and spam.
+ Mark Hammond      17/09/2003  pop3proxy_service: Only munge sys.path in source-code versions, and as the service starts, have it report the username and ini file it is using. In binary builds, write a log to %temp%\SpamBayesServicen.log
+ Mark Hammond      17/09/2003  If we are running Windows, have no valid config file specified, and have the win32all extensions available, default to: \Documents and Settings\[user]\Application Data\SpamBayes\Proxy
+ Tony Meyer        16/09/2003  Fix [ 795145 ] pop3proxy review page dies with mixed gzip/non messages
+ Mark Hammond      15/09/2003  Outlook: Load dialog bitmaps directly from the executable in binary builds.
+ Adam Walker       15/09/2003  Updated pop3proxy_tray to support the service.
+ Tony Meyer        15/09/2003  Removed the gary_combining option and code.
+ Tony Meyer        15/09/2003  Add a (basic) check for version option to the pop3proxy tray app.
+ Tim Peters        15/09/2003  Fix SF bug 806238:  urllib2 fails in Outlook new-version chk.
+ Tim Peters        15/09/2003  Outlook: ShowClues():  Made the clue report a little prettier, and (I hope) a little easier to follow.
+ Tony Meyer        13/09/2003  Fix [ 805351 ] If cc: address is not qualified, proxy fails to send message
+ Mark Hammond      12/09/2003  pop3proxy_tray: Use simple logging strategy similar to the Outlook addin - if we have no console, redirect either to win32traceutil (source-code version) or to a %TEMP\SpamBayesServer1.log (yet to be released binary version).
+ Mark Hammond      12/09/2003  pop3proxy_tray: When running from binary, don't hack sys.path.  When running from source code, hack sys.path based file path rather than on os.getcwd.
+ Mark Hammond      12/09/2003  pop3proxy_tray: When running from binary, load the icon from the executable rather than a .ico file.
+ Tim Peters        12/09/2003  Outlook: ShowClues():  Add lines revealing the # ham and spam trained on.
+ Tony Meyer        11/09/2003  When running setup.py if the old (named) files exist, offer to delete them.
+ Tony Meyer        11/09/2003  Add a unittest to check that we correctly fail when no dbm modules are available.
+ Anthony Baxter    11/09/2003  Add a new file: NEWTRICKS.TXT to record ideas that have and haven't been tried.
+ Richie Hindle     11/09/2003  Bug 803501: Fix the "No dbm modules available" message to print rather than crash. 
+ Skip Montanaro    11/09/2003  Implement a better fix for the storage.py pickle/dbm problems.
+ Mark Hammond      10/09/2003  Outlook: use the classifier's (new) store method rather than an Outlook specific one.
+ Tony Meyer        10/09/2003  Re-fix storage.py so that hammie works with a pickle or dbm.
  Tony Meyer        09/09/2003  Fix for [ 802545 ] crash when loggin off imapfilter UI
  Tony Meyer        09/09/2003  Fix for [ 802347 ] multiline options saved incorrectly



From anadelonbrin at users.sourceforge.net  Thu Sep 18 18:56:11 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 18 18:56:19 2003
Subject: [Spambayes-checkins] 
	spambayes WHAT_IS_NEW.txt, 1.17, 1.18 setup.py, 1.21, 1.22
Message-ID: 

Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv19204

Modified Files:
	WHAT_IS_NEW.txt setup.py 
Log Message:
Install the chkopts script.

Provide information about what's new in the 1.0a6 release.

Bump up a few version numbers.

Index: WHAT_IS_NEW.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/WHAT_IS_NEW.txt,v
retrieving revision 1.17
retrieving revision 1.18
diff -C2 -d -r1.17 -r1.18
*** WHAT_IS_NEW.txt	4 Sep 2003 07:14:57 -0000	1.17
--- WHAT_IS_NEW.txt	18 Sep 2003 22:56:06 -0000	1.18
***************
*** 10,14 ****
  noted in the "Transition" section.
  
! New in Alpha Release 5
  ======================
  
--- 10,14 ----
  noted in the "Transition" section.
  
! New in Alpha Release 6
  ======================
  
***************
*** 17,159 ****
  --------------------------
  
! The values taken by some options have changed, so if you're upgrading from a
! previous version, you may need to update your configuration file (.spambayesrc
! or bayescustomize.ini)
  
!  o allow_remote_connections now takes a list of allowed IP addresses, or the
!    word 'localhost', or an asterisk (meaning all connections are accepted).
!  o notate_to and notate_subject now take a comma-separated list of one or
!    more of 'spam', 'ham' and 'unsure', allowing you to control which classes
!    of message are notated.
  
  Outlook Plugin
  --------------
!  o Added a diagnostics dialog with functions to make it easier for users to
!    help developers track down and fix bugs.
!  o Added a 'timer' method of determining when to filter mail that should work
!    better with Outlook's rule system.
!  o Added a button on the Advanced tab of the dialog to display the SpamBayes
!    data folder.
!  o Moved "Filter Now" to an item on the drop down menu on the toolbar.
!  o Items that can be filtered and trained include "IPM.Note" (normal messages)
!    and "IPM.Anti-Virus*" (virus alerts by some software).
!  o Changed the default filter action to "move" (instead of "untouched").
!  o Added a Wizard to assist with initial configuration (this will present
!    itself when necessary).
!  o Changed to allow filtering to be enabled, even if no training has been done.
!  o Added a "New Folder" button to the folder selector dialog.
!  o Massive changes to the dialog system (which should fix some problems),
!    including changing the configuration dialog to a tabbed interface.
!  o "Show Clues" now shows the percentage, as well as the raw score.
!  o Added a "Help" menu to the drop down menu, with various information.
!  o Added the ability to check for the latest version via an item on the drop
!    down menu.
!  o Hopefully, the "unread flag" issue is now fixed.
!  o Fixed many problems with working on systems where English is not the
!    default language, or where profile names have non-English characters.
  
! POP3 Proxy / SMTP Proxy / POP3 Proxy Service
! --------------------------------------------
!  o Fixed "assert hamcount <= nham" problem.
!  o Starting and stopping the POP3 Proxy service (for Windows NT, Windows
!    2000 and Windows XP users) has been improved.  Most noticeably, this
!    means that the SMTP Proxy will start (if it is needed) as well.
!  o Improve the "notate to" and "notate subject" options, so that ham and
!    unsure messages can also be (optionally) notated in these fields.
!  o Add the ability to skip caching messages that are over a (user
!    configurable) size, so that you can keep the size of the cache
!    directories smaller, once these messages are correctly classified.
!  o Added the ability to skip caching messages that have a precedence of
!    "bulk" (most mailing list messages), so that you can keep the size
!    of the cache directories (and review list) smaller, once these messages
!    are correctly classified.
!  o Fixed the "ASCII decoding error" problem.
!  o The SMTP proxy tries harder to pass on the command formatted exactly
!    as it was given.  This should make it more reliable.
!  o Add the ability to have the SMTP proxy train on the message sent to it,
!    rather than looking up the id in the cache (which is still possible, and
!    generally the better option).
!  o Removed the ability to add the SpamBayes identification number to the
!    body of messages (it can still be added as a header).
!  o The review messages page now puts unsure messages at the top.
!  o The POP3 proxy should now work with fetchmail.
!  o You can once again specify local addresses as well as ports for the
!    POP3 proxy to listen on (was broken in 1.0a3 and 1.0a4).
!  o A bug with the SMTP proxy that would show up in some cases as an
!    "unrecognised command" error the mail client (particularly Eudora)
!    was fixed.
  
  IMAP Filter
  -----------
!  o If you didn't use the -p switch to enter your password
!    interactively, imapfilter would try and get it from the options,
!    but if it wasn't there yet (because you hadn't done the setup yet),
!    it would crash.  This is now fixed.
  
  General
  -------
!  o Added the ability to store the SpamBayes database in a mySQL or
!    postreSGL database table (currently supported by hammiefilter and
!    the POP3 proxy).
!  o Removed the ability to use the 'dumbdbm' as the storage method.
!    (See the FAQ for reasons why).
!  o We now allow the '@' and '=' characters in paths.
!  o Added a simple n-way classifier using a cascade of binary SpamBayes
!    classifiers.
!  o Added version information to the web interface.
!  o Fixed the yellow colour of the header boxes in the web interface.
!  o Fixed restoring defaults from the web interface.
!  o Added a missing line break in the status pane on the web interface
!    when there are no proxies configured.
!  o Prevent the "Show clues" links on the web interface's training page
!    from word-wrapping and making all the table rows two lines high.
!  o You can now put "*" at the end of a word in the "Word Query" box
!    on the web interface, and have it show you the first ten words,
!    and how many words there are in total, in the database that start with
!    that word.
!  o The web interface now supports HTTP-Auth.
!  o Added a new script (code-named 'overkill.py') which enables
!    'drag and drop' training for POP3 users.  This is currently still in
!    the experimental stage, and anyone interested in trying it out should
!    enquire on the SpamBayes mailing list
!    ().
  
  Developer
  ---------
!  o Created a directory for test suites, including a storage.py test.
!  o An empty 'allowed values' now allows an empty string.
!  o Add a get_option method, so an option instance itself can be fetched.
!  o Support fetching the "latest" set of version data from the spambayes
!    web site.
  
  Transition
  ==========
! If you are transitioning from a version older than 1.0a4, please also
  read the notes in the previous release notes (accessible from
  ).
  
!  o If you were previously using the 'dumbdbm' storage method (you will
!    have files called "hammie.db.dat", "hamie.db.dir" and "hammie.db.bak",
!    rather than one file called "hammie.db"), then you will need to
!    change to using either a pickle (please see the FAQ:
!    ), bsddb, gdbm, or one of the new SQL
!    based storage methods.  The 'dumbdbm' storage method resulting in
!    many databases being corrupted, and was never the best choice for
!    storage, in any case.  Although you can use the dbExpImp.py script
!    to convert your database to your new storage system, we recommend
!    that you retrain from scratch, as it is most likely that your
!    database has been corrupted.
!  o If you were using the options to notate the "To" or "Subject" headers
!    with the message's classification, you will need to update your
!    configuration file, as the format for these options have changed.
!  o The ability to add the SpamBayes id to the message body has been
!    removed, which means that Outlook Express users can no longer use
!    the SMTP proxy and have it retrieve messages from the cache.  These
!    users can use the SMTP proxy by training on the forwarded message
!    itself, but this is not recommended, as clues in the message will have
!    changed (the "From" address will be yours, for example).  At this time,
!    you will have to use the web interface for training, although there is
!    the possibility of 'drag and drop' training being added in a release in
!    the near future.
  
  
--- 17,138 ----
  --------------------------
  
! There are two major changes in this release, which *will* effect you if you
! are upgrading from an older version:
  
!  o The scripts have all moved (in the archive), and their names have been
!    changed.  If you run "setup.py install", it will offer to remove the old
!    ones for you, which we recommend.  In the archive, the scripts are all
!    in a 'scripts' directory, and all the scripts start with the "sb_"
!    prefix, to avoid clashing with similiarly named scripts from other
!    packages.  Some name changes go further - "pop3proxy" is now named
!    "sb_server", "hammiefilter" is now named "sb_filter", "hammiecli" is now
!    named "sb_client", "hammiesrv" is now named "sb_xmlrpcserver", "proxytee"
!    is now named "sb_upload", and the experimental "overkill" script is now
!    named "sb_pop3dnd".
! 
! If you were previously using the "hammie.py" script, you will notice that
! it is no longer available.  We recommend that you use either "sb_filter"
! (probably with "sb_mboxtrain"), or use "sb_server" and "sb_upload".  If you
! wish to continue as you were, you can use the "hammie.py" module, which
! will be installed in the "spambayes" package directory, in the same way you
! used the old "hammie.py" script.
! 
!  o All the backwards compatibility code for options which changed names has
!    been removed, which means that you *must* use the correct (new) names.
!    A script (sb_chkopts) is provided which, if you run it, will inform you
!    if you have any invalid names (if will not output anything if there are
!    no problems).
! 
! In addition, the values taken by some options have changed, so if you're
! upgrading from a previous version, you may need to update your configuration
! file (.spambayesrc or bayescustomize.ini)
! 
!  o The options to put the classification in the subject or recipient list
!    (notate_to and notate_subject) have moved from the "pop3proxy" section
!    to the "Headers" section.
!  o All the "pop3proxy" storage options (where the cache is stored, the
!    number of days before messages expire, and so on) have moved to the
!    "Storage" section.
!  o The "hammie" debug header options have been removed, and you should use
!    the "Headers" evidence header options instead.
! 
! Note that pop3proxy (sb_server) and imapfilter users can simply use the web
! interface to check their options and correct any that are wrong.  All
! incorrectlly named options in the configuration file will be removed.
  
  Outlook Plugin
  --------------
!  o Change the default for the ham/spam imbalance adjustment option to
!    False - this should make misclassifications for those with large
!    imbalances easier to understand.  Note that we recommend roughly equal
!    numbers of ham and spam are trained.
!  o Add a warning for those with highly imbalanced ham and spam.
!  o Improved the 'Show Clues' results page.
!  o When we fail to add the 'Spam' field to a read-only store (eg, hotmail),
!    complain less loudly.
  
! POP3 Proxy / SMTP Proxy
! -----------------------
!  o If running Windows, and have the win32 extensions installed, and a
!    configuration file cannot be found, then default to placing it in the
!    "Application Data" directory (as defined by Windows).  Also default to
!    storing the caches and databases in this directory.
!  o Correctly save and close the database when changing options.  This
!    fixes a bug for those using gdbm databases.
! 
! Web Interface
! -------------
!  o Place a threshold on the number of items displayed per section in the
!    review page.
!  o Correctly display the proxy data when it is specified on the command
!    line.
! 
! POP3 Proxy Service / POP3 Proxy Tray Application
! ------------------------------------------------
! 
!  o The POP3 proxy tray application (for Windows users), which quietly
!    appeared in the 1.0a5 release, should be ready for use.  If you have
!    the service installed, it will attempt to use that; if you do not,
!    it will start up sb_server for you.
!  o As the service starts, it reports the username and configuration file
!    that it is using.
!  o Added a version check option to the tray application.
!  o Improved logged for both the service and the tray application.
  
  IMAP Filter
  -----------
!  o Correctly handle IMAP servers that (wrongly) fail to put folder names
!    in quotation marks
!  o Count all messages being classified instead of just the ones from the
!    last folder.
!  o Handle a folder name as a literal when presenting a list to choose from.
!  o Handle imap servers that do not pass a blank result line for an empty
!    search.
!  o Fix IMAP over SSL.
  
  General
  -------
!  o Add a new file: NEWTRICKS.TXT to record ideas that have and haven't
!    been tried.  If you would like to add to this, please submit a patch
!    via the Sourceforge system.
!  o Change the default to tokenizer the "To", "CC", "Reply-To", "Sender",
!    and "From" address headers, rather than just "From".
!  o Fix the database opening code so that hammie* again works with both
!    dbm and pickles.
!  o Minor performance boosts to hammiebulk and mboxtrain.
  
  Developer
  ---------
!  o Add a close method to the various storage classes.
!  o Removed the gary_combining option and code.
  
  Transition
  ==========
! If you are transitioning from a version older than 1.0a5, please also
  read the notes in the previous release notes (accessible from
  ).
  
! Other than the changes listed in "Incompatible changes" at the top of
! this document, there are no known transition issues.
  
  
***************
*** 161,169 ****
  ===================
  The following bugs tracked via the Sourceforge system were fixed:
! 776808, 795749, 787251, 790051, 743131, 779319, 785389, 786952, 788495,
! 790406, 788008, 787296, 788002, 780612, 784323, 784296, 780819, 780801,
! 779049, 765912, 777026, 777165, 693387, 690418, 719586, 769346, 761499,
! 769346, 773452, 765042, 760062, 768162, 768221, 797776, 797316, 796996,
! 797890, 788845 
  
  A url containing the details of these bugs can be made by appending the
--- 140,145 ----
  ===================
  The following bugs tracked via the Sourceforge system were fixed:
! 803501, 802545, 802347, 801952, 798362, 800555, 806632, 795145, 806238, 
! 805351
  
  A url containing the details of these bugs can be made by appending the
***************
*** 174,192 ****
  Feature Requests Added
  ======================
! The following feature requests tracked via the Sourceforge system were
! added:
! 789916, 698036, 796832, 791319
! 
! A url containing the details of these feature requests can be made by
! appending the request number to this url:
! http://sourceforge.net/tracker/index.php?func=detail&group_id=61702&atid=498106&aid=
  
  
  Patches integrated
  ===================
! The following patches tracked via the Sourceforge system were integrated:
! 791254, 790615, 788001, 769981, 791393
! 
! A url containing the details of these feature requests can be made by
! appending the request number to this url:
! http://sourceforge.net/tracker/index.php?func=detail&group_id=61702&atid=498105&aid=
--- 150,159 ----
  Feature Requests Added
  ======================
! No feature requests tracked via the Sourceforge system were added for this
! release.
  
  
  Patches integrated
  ===================
! No patches tracked via the Sourceforge system were integrated for this
! release.

Index: setup.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/setup.py,v
retrieving revision 1.21
retrieving revision 1.22
diff -C2 -d -r1.21 -r1.22
*** setup.py	11 Sep 2003 14:22:38 -0000	1.21
--- setup.py	18 Sep 2003 22:56:06 -0000	1.22
***************
*** 86,89 ****
--- 86,90 ----
               'scripts/sb_upload.py',
               'scripts/sb_xmlrpcserver.py',
+              'scripts/sb_chkopts.py',
              ],
      packages = [



From anadelonbrin at users.sourceforge.net  Thu Sep 18 18:56:13 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 18 18:56:22 2003
Subject: [Spambayes-checkins] spambayes/spambayes Version.py,1.23,1.24
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv19204/spambayes

Modified Files:
	Version.py 
Log Message:
Install the chkopts script.

Provide information about what's new in the 1.0a6 release.

Bump up a few version numbers.

Index: Version.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Version.py,v
retrieving revision 1.23
retrieving revision 1.24
diff -C2 -d -r1.23 -r1.24
*** Version.py	18 Sep 2003 03:58:59 -0000	1.23
--- Version.py	18 Sep 2003 22:56:09 -0000	1.24
***************
*** 28,31 ****
--- 28,33 ----
      # Sub-dict for application specific version strings.
      "Apps": {
+         # Should this be here?  I'm not sure that anything uses it, and the
+         # hammie.py script is gone.  A sb_filter version might be better.
          "Hammie" : {
              "Version":          0.1,
***************
*** 48,54 ****
          },
          "POP3 Proxy" : {
!             "Version":          0.2,
              "BinaryVersion":    0.1,
!             "Description":      "SpamBayes POP3 Proxy Beta2",
              "Date":             "September 2003",
              "InterfaceVersion":            0.03,
--- 50,56 ----
          },
          "POP3 Proxy" : {
!             "Version":          0.3,
              "BinaryVersion":    0.1,
!             "Description":      "SpamBayes POP3 Proxy Beta3",
              "Date":             "September 2003",
              "InterfaceVersion":            0.03,
***************
*** 71,82 ****
          },
          "SMTP Proxy" : {
!             "Version":          0.02,
!             "Description":      "SpamBayes SMTP Proxy Alpha2",
              "Date":             "September 2003",
              "Full Description": "%(Description)s, version %(Version)s (%(Date)s)",
          },
          "IMAP Filter" : {
!             "Version":          0.02,
!             "Description":      "SpamBayes IMAP Filter Alpha2",
              "Date":             "September 2003",
              "InterfaceVersion":            0.02,
--- 73,84 ----
          },
          "SMTP Proxy" : {
!             "Version":          0.1,
!             "Description":      "SpamBayes SMTP Proxy Beta1",
              "Date":             "September 2003",
              "Full Description": "%(Description)s, version %(Version)s (%(Date)s)",
          },
          "IMAP Filter" : {
!             "Version":          0.1,
!             "Description":      "SpamBayes IMAP Filter Beta1",
              "Date":             "September 2003",
              "InterfaceVersion":            0.02,



From mhammond at users.sourceforge.net  Thu Sep 18 20:19:24 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Thu Sep 18 20:19:27 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/installer
	spambayes_addin.iss, 1.11, 1.12
Message-ID: 

Update of /cvsroot/spambayes/spambayes/Outlook2000/installer
In directory sc8-pr-cvs1:/tmp/cvs-serv1549

Modified Files:
	spambayes_addin.iss 
Log Message:
Ooops - forgot to checkin the 0.81 change for the setup script.


Index: spambayes_addin.iss
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/installer/spambayes_addin.iss,v
retrieving revision 1.11
retrieving revision 1.12
diff -C2 -d -r1.11 -r1.12
*** spambayes_addin.iss	8 Sep 2003 08:17:29 -0000	1.11
--- spambayes_addin.iss	19 Sep 2003 00:19:22 -0000	1.12
***************
*** 5,10 ****
  [Setup]
  AppName=Spambayes Outlook Addin
! AppVerName=Spambayes Outlook Addin 0.8
! AppVersion=0.8
  DefaultDirName={pf}\Spambayes Outlook Addin
  DefaultGroupName=Spambayes Outlook Addin
--- 5,10 ----
  [Setup]
  AppName=Spambayes Outlook Addin
! AppVerName=Spambayes Outlook Addin 0.81
! AppVersion=0.81
  DefaultDirName={pf}\Spambayes Outlook Addin
  DefaultGroupName=Spambayes Outlook Addin



From anadelonbrin at users.sourceforge.net  Thu Sep 18 21:04:18 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 18 21:04:21 2003
Subject: [Spambayes-checkins] spambayes/spambayes __init__.py,1.6,1.7
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv9208/spambayes

Modified Files:
	__init__.py 
Log Message:
Release 1.0a6

Index: __init__.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/__init__.py,v
retrieving revision 1.6
retrieving revision 1.7
diff -C2 -d -r1.6 -r1.7
*** __init__.py	4 Sep 2003 21:30:46 -0000	1.6
--- __init__.py	19 Sep 2003 01:04:16 -0000	1.7
***************
*** 1,3 ****
  # package marker.
  
! __version__ = '1.0a5'
--- 1,3 ----
  # package marker.
  
! __version__ = '1.0a6'



From anadelonbrin at users.sourceforge.net  Thu Sep 18 23:45:12 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 18 23:45:16 2003
Subject: [Spambayes-checkins] spambayes WHAT_IS_NEW.txt,1.18,1.19
Message-ID: 

Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv1019

Modified Files:
	WHAT_IS_NEW.txt 
Log Message:
Mark tells me that the tray app will fail right at the top on Win9x (and I preusme ME).
Too much hassle to fix it for 1.0a6, so add a note letting people know that this is the case,
and that it will be resolved in the future.

Index: WHAT_IS_NEW.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/WHAT_IS_NEW.txt,v
retrieving revision 1.18
retrieving revision 1.19
diff -C2 -d -r1.18 -r1.19
*** WHAT_IS_NEW.txt	18 Sep 2003 22:56:06 -0000	1.18
--- WHAT_IS_NEW.txt	19 Sep 2003 03:45:10 -0000	1.19
***************
*** 95,98 ****
--- 95,102 ----
     the service installed, it will attempt to use that; if you do not,
     it will start up sb_server for you.
+    ** Note that the tray application currently only works with Windows
+    NT, Windows 2000 and Windows XP, not Windows 95, 98, or ME.  This
+    will be remedied in a future version, and in the binary release of
+    the tray application. **
   o As the service starts, it reports the username and configuration file
     that it is using.



From mhammond at users.sourceforge.net  Fri Sep 19 00:03:40 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Fri Sep 19 00:03:42 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000 filter.py, 1.32,
	1.33 train.py, 1.33, 1.34
Message-ID: 

Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv3727

Modified Files:
	filter.py train.py 
Log Message:
Remove old dead code from __main__ blocks


Index: filter.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/filter.py,v
retrieving revision 1.32
retrieving revision 1.33
diff -C2 -d -r1.32 -r1.33
*** filter.py	4 Sep 2003 12:14:12 -0000	1.32
--- filter.py	19 Sep 2003 04:03:38 -0000	1.33
***************
*** 143,154 ****
  
  def main():
!     import manager
!     mgr = manager.GetManager()
! 
!     import dialogs.FilterDialog
!     d = dialogs.FilterDialog.FilterArrivalsDialog(mgr, filterer)
!     d.DoModal()
!     mgr.Save()
!     mgr.Close()
  
  if __name__ == "__main__":
--- 143,147 ----
  
  def main():
!     print "Sorry - we don't do anything here any more"
  
  if __name__ == "__main__":

Index: train.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/train.py,v
retrieving revision 1.33
retrieving revision 1.34
diff -C2 -d -r1.33 -r1.34
*** train.py	27 Aug 2003 11:28:47 -0000	1.33
--- train.py	19 Sep 2003 04:03:38 -0000	1.34
***************
*** 187,199 ****
  
  def main():
!     import manager
!     mgr = manager.GetManager()
! 
!     import dialogs.TrainingDialog
!     d = dialogs.TrainingDialog.TrainingDialog(mgr, trainer)
!     d.DoModal()
! 
!     mgr.Save()
!     mgr.Close()
  
  if __name__ == "__main__":
--- 187,191 ----
  
  def main():
!     print "Sorry - we don't do anything here any more"
  
  if __name__ == "__main__":



From mhammond at users.sourceforge.net  Fri Sep 19 01:42:06 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Fri Sep 19 01:42:09 2003
Subject: [Spambayes-checkins] spambayes/windows/py2exe setup_all.py,1.6,1.7
Message-ID: 

Update of /cvsroot/spambayes/spambayes/windows/py2exe
In directory sc8-pr-cvs1:/tmp/cvs-serv17310/py2exe

Modified Files:
	setup_all.py 
Log Message:
First version of a combined binary.


Index: setup_all.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/py2exe/setup_all.py,v
retrieving revision 1.6
retrieving revision 1.7
diff -C2 -d -r1.6 -r1.7
*** setup_all.py	16 Sep 2003 05:21:03 -0000	1.6
--- setup_all.py	19 Sep 2003 05:42:04 -0000	1.7
***************
*** 89,96 ****
  )
  
! outlook_doc_files = [
      ["outlook", [os.path.join(sb_top_dir, r"Outlook2000\about.html")]],
      ["outlook/docs", glob.glob(os.path.join(sb_top_dir, r"Outlook2000\docs\*.html"))],
      ["outlook/docs/images", glob.glob(os.path.join(sb_top_dir, r"Outlook2000\docs\images\*.jpg"))],
  ]
  
--- 89,100 ----
  )
  
! outlook_data_files = [
      ["outlook", [os.path.join(sb_top_dir, r"Outlook2000\about.html")]],
      ["outlook/docs", glob.glob(os.path.join(sb_top_dir, r"Outlook2000\docs\*.html"))],
      ["outlook/docs/images", glob.glob(os.path.join(sb_top_dir, r"Outlook2000\docs\images\*.jpg"))],
+     ["outlook", [os.path.join(sb_top_dir, r"Outlook2000\default_bayes_customize.ini")]],
+ ]
+ proxy_data_files = [
+     ["proxy", [os.path.join(sb_top_dir, r"windows\readme_proxy.html")]],
  ]
  
***************
*** 111,114 ****
        windows=[pop3proxy_tray],
        # and the misc data files
!       data_files = outlook_doc_files,
  )
--- 115,118 ----
        windows=[pop3proxy_tray],
        # and the misc data files
!       data_files = outlook_data_files + proxy_data_files,
  )



From mhammond at users.sourceforge.net  Fri Sep 19 01:42:06 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Fri Sep 19 01:42:11 2003
Subject: [Spambayes-checkins] spambayes/windows readme_proxy.html, NONE,
	1.1 spambayes.iss, NONE, 1.1
Message-ID: 

Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv17310

Added Files:
	readme_proxy.html spambayes.iss 
Log Message:
First version of a combined binary.


--- NEW FILE: readme_proxy.html ---



  
  SpamBayes



        SpamBayes Server/Proxy Application.
        

        
Wel come to SpamBayes
        

        
This SpamBayes application sits between your Mail Server (generally at
your ISP) and your Mail program.  Your mail program still believes
it is talking to a real mail server, but instead it is talking to
SpamBayes, which acts as a proxy
between the two.
        

        
SpamBayes will let all good email through normally.  All Spam or
Unsure items are held by the application for your review.  You
review all messages and configure SpamBayes itself via your web browser.
        

        
If you use Microsoft Outlook, you should probably be using the Outlook
Addin, which was presented as an option at installation time.
        

        

        Getting Started
        
You should start the SpamBayes Tray
Icon program, which was installed on your Start menu.  It
is recommended
        

        

        Using the Service
        
There is a windows service program installed, but it is not configured
by the installation program.  If you configure this manually, you
can continue to use the Tray Icon program, as it will control the
service instead of running the server internally as it normally does.
        
To install the service, perform the following steps:
        

          
  
        • Open a command prompt, and change to the \Program Files\SpamBayes\proxy
directory.
          • 
  
        • Execute pop3proxy_service.exe
-install
          • 
  
        • Select Control
Panel->Administrative Tools->Services, and locate SpamBayes Service in the list.
          • 
  
        • Change the properties of the service so it logs on with your user
account, rather than the builtin system account.  This will ensure
that SpamBayes uses the same configuration and data files when running
as a service and when running as a normal program.
          • 
  
        • If desired, change the properties of the service to start at boot
time.
          • 
  
        • Start the service
          • 
  
        • Start the SpamBayes Tray Icon
program, and confirm the server is running.  Configure and manage
spambayes normally.
          • 

        



--- NEW FILE: spambayes.iss ---
;
; Inno Setup 4.x setup file for the Spambayes Binaries
;

[Setup]
AppName=Spambayes
AppVerName=Spambayes 1.0a6
AppVersion=1.0a6
DefaultDirName={pf}\Spambayes
DefaultGroupName=Spambayes
OutputDir=.
OutputBaseFilename=SpamBayes-Setup
ShowComponentSizes=no
; Note the check for Outlook running has already been done, so no point
; having this file tell them to shutdown outlook!
; Edit file using Windows 'wordpad'
;InfoBeforeFile=installation_notes.rtf
[Files]
Source: "py2exe\dist\lib\*.*"; DestDir: "{app}\lib"; Flags: ignoreversion

Source: "py2exe\dist\outlook\spambayes_addin.dll"; DestDir: "{app}\outlook"; Check: InstallingOutlook; Flags: ignoreversion regserver
Source: "py2exe\dist\outlook\docs\welcome.html"; DestDir: "{app}\outlook\docs"; Check: InstallingOutlook; Flags: isreadme
Source: "py2exe\dist\outlook\*.*"; DestDir: "{app}\outlook"; Check: InstallingOutlook; Flags: ignoreversion recursesubdirs

Source: "py2exe\dist\proxy\*.*"; DestDir: "{app}\proxy"; Check: InstallingProxy; Flags: ignoreversion recursesubdirs
Source: "py2exe\dist\proxy\readme_proxy.html"; DestDir: "{app}\proxy"; Check: InstallingProxy; Flags: isreadme

[Tasks]
Name: startup; Description: "Execute SpamBayes each time Windows starts";
Name: desktop; Description: "Add an icon to the desktop"; Flags: unchecked;

[Run]
FileName:"{app}\proxy\pop3proxy_tray.exe"; Description: "Start the server now"; Flags: postinstall skipifdoesntexist nowait

[Icons]
Name: "{group}\SpamBayes Tray Icon"; Filename: "{app}\proxy\pop3proxy_tray.exe"; Check: InstallingProxy
Name: "{userdesktop}\SpamBayes Tray Icon"; Filename: "{app}\proxy\pop3proxy_tray.exe"; Check: InstallingProxy; Tasks: desktop
Name: "{userstartup}\SpamBayes Tray Icon"; Filename: "{app}\proxy\pop3proxy_tray.exe"; Check: InstallingProxy; Tasks: startup
Name: "{group}\About SpamBayes"; Filename: "{app}\proxy\readme_proxy.html"; Check: InstallingProxy;

Name: "{group}\SpamBayes Outlook Addin\About SpamBayes"; Filename: "{app}\outlook\about.html"; Check: InstallingOutlook
Name: "{group}\SpamBayes Outlook Addin\Troubleshooting Guide"; Filename: "{app}\outlook\docs\troubleshooting.html"; Check: InstallingOutlook

[UninstallDelete]

[Code]
var
  InstallOutlook, InstallProxy: Boolean;
  WarnedNoOutlook, WarnedBoth : Boolean;

function InstallingOutlook() : Boolean;
begin
  Result := InstallOutlook;
end;
function InstallingProxy() : Boolean;
begin
  Result := InstallProxy;
end;

function IsOutlookInstalled() : Boolean;
begin
    Result := RegKeyExists( HKCU, 'Software\Microsoft\Office\Outlook');
end;

function InitializeSetup(): Boolean;
begin
    Result := true;
    while Result do begin
        if not CheckForMutexes('_outlook_mutex_') then
            break;
          Result := MsgBox(
              'You must close Outlook before SpamBayes can be installed.' + #13 + #13 +
              'Please close all Outlook Windows (using "File->Exit and Log off"' + #13 +
              'if available) and click Retry, or click Cancel to exit the installation.'+ #13 + #13 +
              'If this message persists after closing all Outlook windows, you may' + #13 +
              'need to log off from Windows, and try again.',
              mbConfirmation, MB_RETRYCANCEL) = idRetry;
    end;
    // default our install type.
    if IsOutlookInstalled() then begin
      InstallOutlook := True;
      InstallProxy := False
    end
    else begin
      InstallOutlook := False;
      InstallProxy := True;
    end;
end;

// Inno has a pretty primitive "Components/Tasks" concept that
// doesn't quite fit what we want - so we create a custom wizard page.
function PromptApplications( BackClicked: Boolean): Boolean;
var
  Next: Boolean;
  Prompts, Values: array of String;
begin

    // First open the custom wizard page
    ScriptDlgPageOpen();

    // Set some captions
    ScriptDlgPageSetCaption('Select applications to install');
    ScriptDlgPageSetSubCaption1('A number of applications are included with this package.');
    ScriptDlgPageSetSubCaption2('Select the components you wish to install.');

    SetArrayLength(Prompts, 2);
    SetArrayLength(Values, 2);
    if InstallOutlook then
      Prompts[0] := 'Microsoft Outlook Addin (Outlook appears to be installed)'
    else
      Prompts[0] := 'Microsoft Outlook Addin (Outlook does not appear to be installed)';
    Prompts[1] := 'Server/Proxy application, for all other POP based mail clients, including Outlook Express';

    while True do begin
      if InstallOutlook then Values[0] := '1' else Values[0] := '0';
      if InstallProxy then Values[1] := '1' else Values[1] := '0';
      Next:= InputOptionArray(Prompts, Values, False, False);
      if not Next then Break;
      InstallOutlook := (Values[0] = '1');
      InstallProxy := (Values[1] = '1');

      if InstallOutlook and not IsOutlookInstalled and not WarnedNoOutlook then begin
        if MsgBox(
              'Outlook appears to not be installed.' + #13 + #13 +
              'This addin only works with Microsoft Outlook 2000 and later - it' + #13 +
              'does not work with Outlook express.' + #13 + #13 +
              'If you know that Outlook is installed, you may with to continue.' + #13 + #13 +
              'Would you like to change your selection?',
              mbConfirmation, MB_YESNO) = idNo then begin
            WarnedNoOutlook := True;
            Break; // break check loop
          end;
          Continue;
      end;

      if InstallOutlook and InstallProxy and not WarnedBoth then begin
        if MsgBox(
              'You have selected to install both the Outlook Addin and the Server/Proxy Applications.' + #13 + #13 +
              'Unless you regularly use both Outlook and another mailer on the same system' + #13 +
              'you do not need both applications.' + #13 + #13 +
              'Would you like to change your selection?',
              mbConfirmation, MB_YESNO) = idNo then begin
            WarnedBoth := True;
            Break; // break check loop
          end;
        Continue
      end;

      if not InstallOutlook and not InstallProxy then begin
        MsgBox('You must select one of the applications', mbError, MB_OK);
        Continue;
      end
      // we got to here, we are OK.
      Break;
    end
    // See NextButtonClick and BackButtonClick: return True if the click should be allowed
    if not BackClicked then
      Result := Next
    else
      Result := not Next;
    // Close the wizard page. Do a FullRestore only if the click (see above) is not allowed
    ScriptDlgPageClose(not Result);
end;


function ScriptDlgPages(CurPage: Integer; BackClicked: Boolean): Boolean;
begin
  if (not BackClicked and (CurPage = wpWelcome)) or (BackClicked and (CurPage = wpSelectDir)) then begin
    // Insert a custom wizard page between two non custom pages
	Result := PromptApplications( BackClicked );
  end
  else
    Result := True;
end;

function NextButtonClick(CurPage: Integer): Boolean;
begin
  Result := ScriptDlgPages(CurPage, False);
end;

function BackButtonClick(CurPage: Integer): Boolean;
begin
  Result := ScriptDlgPages(CurPage, True);
end;

function SkipCurPage(CurPage: Integer): Boolean;
begin
  Result := (CurPage = wpSelectTasks) and (not InstallProxy);
end;



From anadelonbrin at users.sourceforge.net  Fri Sep 19 03:18:28 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 19 03:18:30 2003
Subject: [Spambayes-checkins] spambayes setup.py,1.22,1.23
Message-ID: 

Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv1482

Modified Files:
	setup.py 
Log Message:
If we are removing scripts with the old names, remove any pyc or pyo variants as well,
but quietly.

Index: setup.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/setup.py,v
retrieving revision 1.22
retrieving revision 1.23
diff -C2 -d -r1.22 -r1.23
*** setup.py	18 Sep 2003 22:56:06 -0000	1.22
--- setup.py	19 Sep 2003 07:18:26 -0000	1.23
***************
*** 60,63 ****
--- 60,70 ----
                      try:
                          os.remove(s)
+                         # Also remove .pyc and .pyo, but quietly.
+                         pyc = "%sc" % (s,)
+                         pyo = "%so" % (s,)
+                         if os.path.exists(pyc):
+                             os.remove(pyc)
+                         if os.path.exists(pyo):
+                             os.remove(pyo)
                          print "Removed", s
                      except OSError:



From anadelonbrin at users.sourceforge.net  Fri Sep 19 03:51:08 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 19 03:51:11 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_smtpproxy.py,1.2,1.3
Message-ID: 

Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv8412/scripts

Modified Files:
	sb_smtpproxy.py 
Log Message:
How this slipped past earlier tesing, I have no idea.
Fix an incorrect name.

Index: sb_smtpproxy.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_smtpproxy.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** sb_smtpproxy.py	13 Sep 2003 04:03:41 -0000	1.2
--- sb_smtpproxy.py	19 Sep 2003 07:51:06 -0000	1.3
***************
*** 370,374 ****
      
      def extractSpambayesID(self, data):
!         msg = message_from_string(data)
  
          # The nicest MUA is one that forwards the header intact.
--- 370,374 ----
      
      def extractSpambayesID(self, data):
!         msg = sbheadermessage_from_string(data)
  
          # The nicest MUA is one that forwards the header intact.



From anadelonbrin at users.sourceforge.net  Fri Sep 19 04:10:28 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 19 04:10:35 2003
Subject: [Spambayes-checkins] spambayes/windows readme_proxy.html,1.1,1.2
Message-ID: 

Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv13583/windows

Modified Files:
	readme_proxy.html 
Log Message:
Tidy up the documentation a bit, and correct a few mistakes.  It was a nice attempt for
someone that doesn't use the proxy, though .

Note that the image links don't work at the moment.  I'm not sure where they should try and
find the images, since they're in the Outlook stuff at the moment.

Index: readme_proxy.html
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/readme_proxy.html,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** readme_proxy.html	19 Sep 2003 05:42:03 -0000	1.1
--- readme_proxy.html	19 Sep 2003 08:10:25 -0000	1.2
***************
*** 7,38 ****
  
  
! 
        SpamBayes Server/Proxy Application.
        
! 
        
! Wel come to SpamBayes
        
! 
        
! This SpamBayes application sits between your Mail Server (generally at
! your ISP) and your Mail program.  Your mail program still believes
  it is talking to a real mail server, but instead it is talking to
  SpamBayes, which acts as a proxy
! between the two.
        
! 
        
! SpamBayes will let all good email through normally.  All Spam or
! Unsure items are held by the application for your review.  You
! review all messages and configure SpamBayes itself via your web browser.
        
! 
        
! If you use Microsoft Outlook, you should probably be using the Outlook
! Addin, which was presented as an option at installation time.
        
! 
        
  
        Getting Started
        
! You should start the SpamBayes Tray
! Icon program, which was installed on your Start menu.  It
! is recommended
        
! 
        
  
        Using the Service
        
! There is a windows service program installed, but it is not configured
! by the installation program.  If you configure this manually, you
  can continue to use the Tray Icon program, as it will control the
! service instead of running the server internally as it normally does.
        
! To install the service, perform the following steps:
        
  
          
    
        • Open a command prompt, and change to the 
  
! 
!   
!     
!       
! 
!       
!       
!     
!   
! 
          Logo 
          
! 
          SpamBayes Server/Proxy Application
          
! 
          Welcome to SpamBayes
          
! 
! 
          Note that if you use Microsoft Outlook, you should probably be using the
! Outlook Addin, which was presented as an option at installation time.
          
! 
! 
          This SpamBayes application sits between your mail server (generally at
! your ISP) and your mail program. Your mail program still believes
  it is talking to a real mail server, but instead it is talking to
  SpamBayes, which acts as a proxy
! between the two.
          
! 
! 
          All your mail arrives as normal in your mail program, but, as it arrives,
! SpamBayes adds a classification to the messages - either ham (good
! mail), spam (bad mail), or unsure (messages that SpamBayes isn't
! certain about).  You can set your mail program up to automatically
! filter messages into folders based on this classification - for example you
! might move all spam messages to a folder called "Junk Mail", and all
! unsure messages to a folder called "Mail to review".
          
! 
! 
          In order to classify messages, you have to train SpamBayes.  The
! easiest method of doing this is via an interface that SpamBayes presents
! to you via your web browser.  SpamBayes will list all the messages that have
! been received, and let you train them as ham or spam.  You
! can also configure SpamBayes via this interface in your web browser.
          
! 
  
          Getting Started
          
! 
          You should start the SpamBayes Tray
! Icon program, which was installed into your Start menu. This is the
! recommended method of operating SpamBayes.  If you right-click on this
! icon you will be presented with a number of options to control SpamBayes,
! including an option to configure SpamBayes for your system.
          
! 
  
          Using the Service
          
! 
          There is a windows service program installed, but it is not configured
! by the installation program. If you configure this manually, you
  can continue to use the Tray Icon program, as it will control the
! service instead of running the server internally as it normally does.
          
! 
          To install the service, perform the following steps:
          
  
            
    
          • Open a command prompt, and change to the SpamBayes Service in the list.
            • 
    
          • Change the properties of the service so it logs on with your user
! account, rather than the builtin system account.  This will ensure
  that SpamBayes uses the same configuration and data files when running
  as a service and when running as a normal program.
            • 
--- 68,72 ----
   style="font-style: italic;">SpamBayes Service in the list.
    
          • Change the properties of the service so it logs on with your user
! account, rather than the builtin system account. This will ensure
  that SpamBayes uses the same configuration and data files when running
  as a service and when running as a normal program.
            • 
***************
*** 51,56 ****
  time.
    
          • Start the service
            • 
!   
          • Start the SpamBayes Tray Icon
! program, and confirm the server is running.  Configure and manage
  spambayes normally.
            • 
  
          
--- 74,79 ----
  time.
          • 
    
        • Start the service
          • 
!   
        • Start the SpamBayes Tray Icon
! program, and confirm the server is running. Configure and manage
  spambayes normally.
          • 
  
        



From anadelonbrin at users.sourceforge.net  Fri Sep 19 04:24:12 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 19 04:24:17 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/docs welcome.html, 1.5,
	1.6
Message-ID: 

Update of /cvsroot/spambayes/spambayes/Outlook2000/docs
In directory sc8-pr-cvs1:/tmp/cvs-serv16508/Outlook2000/docs

Modified Files:
	welcome.html 
Log Message:
Tidy up the welcome doc a bit (mostly internally, hardly any external change).



Index: welcome.html
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/docs/welcome.html,v
retrieving revision 1.5
retrieving revision 1.6
diff -C2 -d -r1.5 -r1.6
*** welcome.html	10 Sep 2003 07:38:51 -0000	1.5
--- welcome.html	19 Sep 2003 08:24:10 -0000	1.6
***************
*** 12,17 ****
        Logo
!   
        
--- 12,17 ----
        Logo
! 
!        
        
***************
*** 19,38 ****
    
  
! 
        Welcome to SpamBayes 
        
! 
        
  
        SpamBayes is an Outlook plugin that provides a spam filter based on
  statistical analysis of
! your personal mail.  Unlike many other spam detection systems,
  SpamBayes
  actually learns what you consider spam, and continually adapts as both
  your regular email and spam patterns change.
        
  
        When you first start Outlook after SpamBayes has been installed, the
! SpamBayes Installation Wizard will appear.  This Wizard will guide
  you through the configuration process and allow you to quickly have
! SpamBayes filtering your mail.  This document contains additional
  information which will help make SpamBayes effective from the first
  time you use it.
        
  
        Please remember that this is free software; please be patient, and
! note that there are plenty of things to improve.  There are ways
  you can help, even if you aren't a programmer - you could help with
  this documentation, or 
  
! 
        Welcome to SpamBayes
        
  
        SpamBayes is an Outlook plugin that provides a spam filter based on
  statistical analysis of
! your personal mail. Unlike many other spam detection systems,
  SpamBayes
  actually learns what you consider spam, and continually adapts as both
  your regular email and spam patterns change.
        
  
        When you first start Outlook after SpamBayes has been installed, the
! SpamBayes Installation Wizard will appear. This Wizard will guide
  you through the configuration process and allow you to quickly have
! SpamBayes filtering your mail. This document contains additional
  information which will help make SpamBayes effective from the first
  time you use it.
        
  
        Please remember that this is free software; please be patient, and
! note that there are plenty of things to improve. There are ways
  you can help, even if you aren't a programmer - you could help with
  this documentation, or main project
! page for information.  The list of Frequently Asked Questions
  may also answer some of yours.
        
--- 39,43 ----
  any number of other ways - please see the main project
! page for information. The list of Frequently Asked Questions
  may also answer some of yours.
        
***************
*** 49,67 ****
  rules, so anything you consider spam will be treated as spam by this
  system, even if it does not conform to the traditional definitions of
! spam.  This means that SpamBayes requires training before it will be
! effective.  There are two ways that this training can be done:
        
  
          
!   
        • Allow SpamBayes to learn as it goes.  Initially, SpamBayes
  will consider all mail items unsure,
  and each item will be used to
! train.  In this scenario, SpamBayes will take some time to become
! effective.  It will rarely make mistakes, but will continue to be
  unsure about items until the training information grows.
          
    
          • 
    
        • Pre-sorting mail into two folders; one containing only examples
! of good messages, and another containing only examples of spam. 
  SpamBayes will then process all these messages gathering the clues it
! uses to filter mail..  Depending on how many messages you train
  on, SpamBayes will be immediately effective at correctly classifying
  your mail.
          • 
--- 48,66 ----
  rules, so anything you consider spam will be treated as spam by this
  system, even if it does not conform to the traditional definitions of
! spam. This means that SpamBayes requires training before it will be
! effective. There are two ways that this training can be done:
          
  
            
!   
          • Allow SpamBayes to learn as it goes. Initially, SpamBayes
  will consider all mail items unsure,
  and each item will be used to
! train. In this scenario, SpamBayes will take some time to become
! effective. It will rarely make mistakes, but will continue to be
  unsure about items until the training information grows.
            
    
            • 
    
          • Pre-sorting mail into two folders; one containing only examples
! of good messages, and another containing only examples of spam.
  SpamBayes will then process all these messages gathering the clues it
! uses to filter mail. Depending on how many messages you train
  on, SpamBayes will be immediately effective at correctly classifying
  your mail.
            • 
***************
*** 70,76 ****
  
            It is important to note that even when SpamBayes has little training
  information, it rarely gets things wrong - the worst it generally does
! is to classify a message as unsure. 
  However, as mentioned, the more training information SpamBayes has, the
! less it is unsure about new messages.  See using the plugin below
  for more information.
            
  
            Using the Plugin
            
--- 69,75 ----
  
            It is important to note that even when SpamBayes has little training
  information, it rarely gets things wrong - the worst it generally does
! is to classify a message as unsure.
  However, as mentioned, the more training information SpamBayes has, the
! less it is unsure about new messages. See using the plugin below
  for more information.
            
  
            Using the Plugin
            
***************
*** 80,84 ****
      
        This section describes how the
! plugin operates once it is configured. 
  You can access the SpamBayes features from the SpamBayes toolbar (shown
  to the right), but
--- 79,83 ----
      
        This section describes how the
! plugin operates once it is configured.
  You can access the SpamBayes features from the SpamBayes toolbar (shown
  to the right), but
***************
*** 88,92 ****
        SpamBayes Toolbar
            
        
      
--- 87,91 ----
        SpamBayes Toolbar
            
        
      
***************
*** 95,105 ****
  
            
  
            As messages arrive, they are given a spam score.  This score is
  the measure of how "spammy" the system has decided a mail is, with 100%
  being certain spam, and 0% meaning the message is certainly not
! spam.  The SpamBayes addin uses these scores to classify mail into
  one of three categories -
  certain spam, unsure, and good messages. 
  Good messages are often
  known in the anti-spam community as ham.
            
--- 94,104 ----
  
            
  
            As messages arrive, they are given a spam score. This score is
  the measure of how "spammy" the system has decided a mail is, with 100%
  being certain spam, and 0% meaning the message is certainly not
! spam. The SpamBayes addin uses these scores to classify mail into
  one of three categories -
  certain spam, unsure, and good messages.
  Good messages are often
  known in the anti-spam community as ham.
            
***************
*** 107,111 ****
  they
  will remain in your inbox, or be filtered normally by Outlook's builtin
! rules.  Mail that is classified as either unsure or certain spam
--- 106,110 ----
  they
  will remain in your inbox, or be filtered normally by Outlook's builtin
! rules. Mail that is classified as either unsure or certain spam
***************
*** 114,131 ****
   style="font-style: italic;">spam score to your Outlook folder
  views - but you can do it
! manually by following these instructions.
            
  
            
  There are three ways in which the system can get things wrong:
  
              
!   
            • A spam stays in your inbox.  This is known as a false negative.  In this case
  you can either drag the message to the Spam folder or click on the Delete as Spam button on the
  Outlook
! toolbar.  In both cases, the message will be trained as spam and
  will be moved to the spam folder.
              • 
!   
            • Any message is moved to the unsure folder.  In this case,
  the system is simply unsure about the message, and moves it to the
! possible spam folder for human review.  All unsure messages should
  be manually classified; good messages can either be dragged back to the
  inbox, or have the Recover from Spam
--- 113,130 ----
   style="font-style: italic;">spam score to your Outlook folder
  views - but you can do it
! manually by following these instructions.
              
  
              
  There are three ways in which the system can get things wrong:
  
                
!   
              • A spam stays in your inbox. This is known as a false negative. In this case
  you can either drag the message to the Spam folder or click on the Delete as Spam button on the
  Outlook
! toolbar. In both cases, the message will be trained as spam and
  will be moved to the spam folder.
                • 
!   
              • Any message is moved to the unsure folder. In this case,
  the system is simply unsure about the message, and moves it to the
! possible spam folder for human review. All unsure messages should
  be manually classified; good messages can either be dragged back to the
  inbox, or have the Recover from Spam
***************
*** 134,155 ****
  Spam folder or have the Delete as
  Spam
! toobar button (shown above) clicked.  In all cases, the system
  will
  automatically
  be trained appropriately.
                • 
    
              • A wanted (ham) message is moved to the Spam folder. This is known
! as a false positive. In
  this case you can either drag the message back to the folder from which
  it came (generally the inbox), or click on the Recover from Spam button.  In
  both cases the message will be trained as good, and moved back to the
  original folder.
                • 
  
              
  Note that in all cases, as you take corrective action on the mail, the
! system is also trained.  This makes it less likely that another
! similar mail will be incorrectly classified in the future.
              
! 
              
! 
              
  
  
--- 133,153 ----
  Spam folder or have the Delete as
  Spam
! toobar button (shown above) clicked. In all cases, the system
  will
  automatically
  be trained appropriately.
              • 
    
            • A wanted (ham) message is moved to the Spam folder. This is known
! as a false positive. In
  this case you can either drag the message back to the folder from which
  it came (generally the inbox), or click on the Recover from Spam button. In
  both cases the message will be trained as good, and moved back to the
  original folder.
              • 
  
            
  Note that in all cases, as you take corrective action on the mail, the
! system is also trained. This makes it less likely that another
! similar mail will be incorrectly classified in the future.
            
! 
            
! 
            
  
  



From anadelonbrin at users.sourceforge.net  Fri Sep 19 04:24:56 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 19 04:24:59 2003
Subject: [Spambayes-checkins] 
	spambayes/spambayes/test test_sb-server.py, 1.1, 1.2
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes/test
In directory sc8-pr-cvs1:/tmp/cvs-serv16637/spambayes/test

Modified Files:
	test_sb-server.py 
Log Message:
Fix the sb_server test so that it again runs, and passes.

Index: test_sb-server.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/test/test_sb-server.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** test_sb-server.py	5 Sep 2003 01:16:46 -0000	1.1
--- test_sb-server.py	19 Sep 2003 08:24:54 -0000	1.2
***************
*** 1,5 ****
  #! /usr/bin/env python
  
! """Test the pop3proxy is working correctly.
  
  When using the -z command line option, carries out a test that the
--- 1,5 ----
  #! /usr/bin/env python
  
! """Test the POP3 proxy is working correctly.
  
  When using the -z command line option, carries out a test that the
***************
*** 83,92 ****
  import getopt
  
! # a bit of a hack to help those without spambayes on their
! # Python path - stolen from timtest.py
  import sys
  import os
  sys.path.insert(-1, os.getcwd())
! sys.path.insert(-1, os.path.dirname(os.getcwd()))
  
  from spambayes import Dibbler
--- 83,93 ----
  import getopt
  
! # We need to import sb_server, but it may not be on the PYTHONPATH.
! # Hack around this, so that if we are running in a cvs-like setup
! # everything still works.
  import sys
  import os
  sys.path.insert(-1, os.getcwd())
! sys.path.insert(-1, os.path.join(os.getcwd(), "scripts"))
  
  from spambayes import Dibbler
***************
*** 94,99 ****
  from spambayes.UserInterface import UserInterfaceServer
  from spambayes.ProxyUI import ProxyUserInterface
! from pop3proxy import BayesProxyListener
! from pop3proxy import state, _recreateState
  from spambayes.Options import options
  
--- 95,100 ----
  from spambayes.UserInterface import UserInterfaceServer
  from spambayes.ProxyUI import ProxyUserInterface
! from sb_server import BayesProxyListener
! from sb_server import state, _recreateState
  from spambayes.Options import options
  
***************
*** 101,105 ****
  # is added to the size of each message.
  HEADER_EXAMPLE = '%s: xxxxxxxxxxxxxxxxxxxx\r\n' % \
!                  options["Hammie", "header_name"]
  
  class TestListener(Dibbler.Listener):
--- 102,106 ----
  # is added to the size of each message.
  HEADER_EXAMPLE = '%s: xxxxxxxxxxxxxxxxxxxx\r\n' % \
!                  options["Headers", "classification_header_name"]
  
  class TestListener(Dibbler.Listener):
***************
*** 300,304 ****
          while response.find('\n.\r\n') == -1:
              response = response + proxy.recv(1000)
!         assert response.find(options["Hammie", "header_name"]) >= 0
  
      # Smoke-test the HTML UI.
--- 301,305 ----
          while response.find('\n.\r\n') == -1:
              response = response + proxy.recv(1000)
!         assert response.find(options["Headers", "classification_header_name"]) >= 0
  
      # Smoke-test the HTML UI.



From anadelonbrin at users.sourceforge.net  Fri Sep 19 04:28:09 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 19 04:28:12 2003
Subject: [Spambayes-checkins] 
	spambayes/spambayes/test test_sb-server.py, 1.2, 1.3
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes/test
In directory sc8-pr-cvs1:/tmp/cvs-serv17219/spambayes/test

Modified Files:
	test_sb-server.py 
Log Message:
Comment typo.

Index: test_sb-server.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/test/test_sb-server.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** test_sb-server.py	19 Sep 2003 08:24:54 -0000	1.2
--- test_sb-server.py	19 Sep 2003 08:28:07 -0000	1.3
***************
*** 8,12 ****
  web ui is present.
  
! The -t option runs a fake POP3 server on port 8810.  This is the
  same server that the -z option uses, and may be separately run for
  other testing purposes.
--- 8,12 ----
  web ui is present.
  
! The -t option runs a fake POP3 server on port 8110.  This is the
  same server that the -z option uses, and may be separately run for
  other testing purposes.



From montanaro at users.sourceforge.net  Fri Sep 19 09:14:23 2003
From: montanaro at users.sourceforge.net (Skip Montanaro)
Date: Fri Sep 19 09:14:30 2003
Subject: [Spambayes-checkins] spambayes setup.py,1.23,1.24
Message-ID: 

Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv11396

Modified Files:
	setup.py 
Log Message:
Be more vocal about removing pyc's and pyo's.  In particular, people may
have already run the earlier version of setup.py which discarded the py's
and left the pyc's and pyo's.

I'm not convinced that any of these scripts would have been imported as
modules from the bin directory (I didn't have any leftover pycs), but I'll
take Richie's word for it. ;-)


Index: setup.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/setup.py,v
retrieving revision 1.23
retrieving revision 1.24
diff -C2 -d -r1.23 -r1.24
*** setup.py	19 Sep 2003 07:18:26 -0000	1.23
--- setup.py	19 Sep 2003 13:14:20 -0000	1.24
***************
*** 31,46 ****
  class install_scripts(parent):
      old_scripts=[
!         'unheader.py',
!         'hammie.py',
!         'hammiecli.py',
!         'hammiesrv.py',
!         'hammiefilter.py',
!         'pop3proxy.py',
!         'smtpproxy.py',
!         'proxytee.py',
!         'dbExpImp.py',
!         'mboxtrain.py',
!         'imapfilter.py',
!         'notesfilter.py',
          ]
  
--- 31,46 ----
  class install_scripts(parent):
      old_scripts=[
!         'unheader',
!         'hammie',
!         'hammiecli',
!         'hammiesrv',
!         'hammiefilter',
!         'pop3proxy',
!         'smtpproxy',
!         'proxytee',
!         'dbExpImp',
!         'mboxtrain',
!         'imapfilter',
!         'notesfilter',
          ]
  
***************
*** 49,55 ****
          for s in self.old_scripts:
              s = os.path.join(self.install_dir, s)
!             if os.path.exists(s):
!                 print >> sys.stderr, "Error: old script", s, "still exists."
!                 err = True
          if err:
              print >>sys.stderr, "Do you want to delete these scripts? (y/n)"
--- 49,57 ----
          for s in self.old_scripts:
              s = os.path.join(self.install_dir, s)
!             for e in (".py", ".pyc", ".pyo"):
!                 if os.path.exists(s+e):
!                     print >> sys.stderr, "Error: old script", s+e,
!                     print >> sys.stderr, "still exists."
!                     err = True
          if err:
              print >>sys.stderr, "Do you want to delete these scripts? (y/n)"
***************
*** 58,73 ****
                  for s in self.old_scripts:
                      s = os.path.join(self.install_dir, s)
!                     try:
!                         os.remove(s)
!                         # Also remove .pyc and .pyo, but quietly.
!                         pyc = "%sc" % (s,)
!                         pyo = "%so" % (s,)
!                         if os.path.exists(pyc):
!                             os.remove(pyc)
!                         if os.path.exists(pyo):
!                             os.remove(pyo)
!                         print "Removed", s
!                     except OSError:
!                         pass
          return parent.run(self)
  
--- 60,69 ----
                  for s in self.old_scripts:
                      s = os.path.join(self.install_dir, s)
!                     for e in (".py", ".pyc", ".pyo"):
!                         try:
!                             os.remove(s+e)
!                             print "Removed", s+e
!                         except OSError:
!                             pass
          return parent.run(self)
  



From anadelonbrin at users.sourceforge.net  Fri Sep 19 18:47:15 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 19 18:47:20 2003
Subject: [Spambayes-checkins] 
	spambayes/spambayes/test test_sb-server.py, 1.3, 1.4
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes/test
In directory sc8-pr-cvs1:/tmp/cvs-serv3518/spambayes/test

Modified Files:
	test_sb-server.py 
Log Message:
Another comment typo.

Index: test_sb-server.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/test/test_sb-server.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** test_sb-server.py	19 Sep 2003 08:28:07 -0000	1.3
--- test_sb-server.py	19 Sep 2003 22:47:13 -0000	1.4
***************
*** 14,18 ****
  Usage:
  
!     pop3proxytest.py [options]
  
          options:
--- 14,18 ----
  Usage:
  
!     test_sb-server.py [options]
  
          options:



From anadelonbrin at users.sourceforge.net  Fri Sep 19 18:48:31 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 19 18:48:34 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py,1.11,1.12
Message-ID: 

Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv3734/windows

Modified Files:
	pop3proxy_tray.py 
Log Message:
While explaining how to use the tray app to my fiance, I realised that the most common
operation is probably going to be reviewing messages, so add that as a menu item.

Index: pop3proxy_tray.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_tray.py,v
retrieving revision 1.11
retrieving revision 1.12
diff -C2 -d -r1.11 -r1.12
*** pop3proxy_tray.py	15 Sep 2003 04:48:50 -0000	1.11
--- pop3proxy_tray.py	19 Sep 2003 22:48:29 -0000	1.12
***************
*** 115,122 ****
          self.control_functions = {START_STOP_ID : ("Stop SpamBayes", self.StartStop),
                                    1025 : ("-", None),
!                                   1026 : ("View information ...", self.OpenInterface),
!                                   1027 : ("Configure ...", self.OpenConfig),
!                                   1028 : ("Check for latest version", self.CheckVersion),
!                                   1029 : ("-", None),
                                    1099 : ("Exit SpamBayes", self.OnExit),
                                    }
--- 115,123 ----
          self.control_functions = {START_STOP_ID : ("Stop SpamBayes", self.StartStop),
                                    1025 : ("-", None),
!                                   1026 : ("Review messages ...", self.OpenReview),
!                                   1027 : ("View information ...", self.OpenInterface),
!                                   1028 : ("Configure ...", self.OpenConfig),
!                                   1029 : ("Check for latest version", self.CheckVersion),
!                                   1030 : ("-", None),
                                    1099 : ("Exit SpamBayes", self.OnExit),
                                    }
***************
*** 410,413 ****
--- 411,421 ----
          if self.started:
              webbrowser.open_new("http://localhost:%d/config" % \
+                                 (options["html_ui", "port"],))
+         else:
+             self.ShowMessage("SpamBayes is not running.")
+             
+     def OpenReview(self):		
+         if self.started:
+             webbrowser.open_new("http://localhost:%d/review" % \
                                  (options["html_ui", "port"],))
          else:



From anadelonbrin at users.sourceforge.net  Fri Sep 19 19:12:55 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 19 19:12:59 2003
Subject: [Spambayes-checkins] 
	website applications.ht, 1.22, 1.23 download.ht,
	1.18, 1.19 index.ht, 1.26, 1.27
Message-ID: 

Update of /cvsroot/spambayes/website
In directory sc8-pr-cvs1:/tmp/cvs-serv7774

Modified Files:
	applications.ht download.ht index.ht 
Log Message:
Release 1.0a6.

Index: applications.ht
===================================================================
RCS file: /cvsroot/spambayes/website/applications.ht,v
retrieving revision 1.22
retrieving revision 1.23
diff -C2 -d -r1.22 -r1.23
*** applications.ht	8 Sep 2003 10:55:58 -0000	1.22
--- applications.ht	19 Sep 2003 23:12:52 -0000	1.23
***************
*** 11,22 ****
  
            Outlook
            
  
            Sean True and Mark Hammond have developed an addin for Outlook (2000 and XP) that
! adds support for the spambayes classifier. 
  
            Requirements
            
  
              
  
            • Outlook 2000 or Outlook XP (not Outlook Express)
              • 
  
            
! 
            If you are going to use a source code version, you will also need:
            
  
              
! 
            • Python 2.2 or later (2.2.3 recommended)
              • 
  
            • Python's win32com
  extensions (win32all-149 or later - currently ActivePython is not suitable)
--- 11,22 ----
  
              Outlook
              
  
              Sean True and Mark Hammond have developed an addin for Outlook (2000 and XP) that
! adds support for the spambayes classifier.
              
  
              Requirements
              
  
                
  
              • Outlook 2000 or Outlook XP (not Outlook Express)
                • 
  
              
! 
              If you are going to use a source code version, you will also need:
  
                
! 
              • Python 2.2 or later (2.3 recommended)
                • 
  
              • Python's win32com
  extensions (win32all-149 or later - currently ActivePython is not suitable)
***************
*** 29,83 ****
  
                Mark has packaged together an installer for the plugin.
  Go to the Windows page for more.
                
! 
                Download the alpha4 release.
                
  
                Alternatively, you can use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
                
  
! 
                hammiefilter.py
                
! 
                hammie is a command line tool for marking mail as ham or spam. Skip Montanaro has started a
  guide to integrating hammie with your mailer (Unix-only instructions at the moment - additions welcome!). 
! Currently it focusses on running hammie via procmail. 
                
  
  
                Requirements
                
  
                  
! 
                • Python 2.2 or later (2.2.3 recommended)
                  • 
  
                • Currently documentation focuses on Unix.
                  • 
  
                
  
  
                Availability
                
! 
                Download the alpha5 release.
                
  
                Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
                
  
! 
                pop3proxy.py
                
! 
                pop3proxy sits between your mail client and your real POP3 server and marks
! mail as ham or spam as it passes through. See the docstring at the top of pop3proxy.py for more.
  
  
                Requirements
                
  
                  
! 
                • Python 2.2 or later (2.2.3 recommended)
                  • 
  
                • Should work on windows/unix/whatever... 
                  • 
  
                
- 
                
  
  
                Availability
                
! 
                Download the alpha4 release.
                
  
                Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
                
  
! 
                imapfilter.py
                
! 
                imap filter connects to your imap server and marks mail as ham or spam,
  moving it to appropriate folders as it arrives.
! See the docstring at the top of imapfilter.py for more.
                
! 
                Note that this application is 'more' alpha than the others.
                
  
  
                Requirements
                
  
                  
! 
                • Python 2.2 or later (2.2.3 recommended)
                  • 
  
                • Should work on windows/unix/whatever...
                  • 
  
                
- 
                
  
  
                Availability
                
! 
                Download the alpha5 release.
                
  
                Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
                
  
! 
                mboxtrain.py
                
  
                This application allows you to train incrementally on ham and spam
  messages stored in Unix mailbox format files.  When run, it adds an extra
--- 29,81 ----
  
                Mark has packaged together an installer for the plugin.
  Go to the Windows page for more.
                
! 
                Download the alpha6 release.
                
  
                Alternatively, you can use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
                
  
! 
                sb_filter.py
                
! 
                sb_filter is a command line tool for marking mail as ham or spam. Skip Montanaro has started a
  guide to integrating hammie with your mailer (Unix-only instructions at the moment - additions welcome!). 
! Currently it focusses on running sb_filter via procmail. 
                
  
  
                Requirements
                
  
                  
! 
                • Python 2.2 or later (2.3 recommended)
                  • 
  
                • Currently documentation focuses on Unix.
                  • 
  
                
  
  
                Availability
                
! 
                Download the alpha6 release.
                
  
                Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
                
  
! 
                sb_server.py
                
! 
                sb_server provides a POP3 proxy which sits between your mail client and your real POP3 server and marks
! mail as ham or spam as it passes through. See the README for more.
! sb_server can also be used with the sb_upload.py script as a procmail (or similar) solution.
                
  
  
                Requirements
                
  
                  
! 
                • Python 2.2 or later (2.3 recommended)
                  • 
  
                • Should work on windows/unix/whatever... 
                  • 
  
                
  
  
                Availability
                
! 
                Download the alpha6 release.
                
  
                Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
                
  
! 
                sb_imapfilter.py
                
! 
                imapfilter connects to your imap server and marks mail as ham or spam,
  moving it to appropriate folders as it arrives.
! See the README for more.
                
  
  
                Requirements
                
  
                  
! 
                • Python 2.2 or later (2.3 recommended)
                  • 
  
                • Should work on windows/unix/whatever...
                  • 
  
                
  
  
                Availability
                
! 
                Download the alpha6 release.
                
  
                Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
                
  
! 
                sb_mboxtrain.py
                
  
                This application allows you to train incrementally on ham and spam
  messages stored in Unix mailbox format files.  When run, it adds an extra
***************
*** 87,99 ****
  
                Requirements
                
  
                  
! 
                • Python 2.2 or later (2.2.3 recommended)
                  • 
  
                • The code is currently Unix-specific.
                  • 
  
                
  
  
                Availability
                
! 
                Download the alpha5 release.
                
  
                Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
                
  
! 
                notesfilter.py
                
  
                This application allows you to filter Lotus Notes folders, rather like
  the IMAP filter works, moving mail to the correct folder as it arrives.
--- 85,97 ----
  
                Requirements
                
  
                  
! 
                • Python 2.2 or later (2.3 recommended)
                  • 
  
                • The code is currently Unix-specific.
                  • 
  
                
  
  
                Availability
                
! 
                Download the alpha6 release.
                
  
                Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
                
  
! 
                sb_notesfilter.py
                
  
                This application allows you to filter Lotus Notes folders, rather like
  the IMAP filter works, moving mail to the correct folder as it arrives.
***************
*** 105,113 ****
  
                Requirements
                
  
                  
! 
                • Python 2.2 or later (2.2.3 recommended)
                  • 
  
                • Should work on windows/unix/whatever...
                  • 
  
                
  
  
                Availability
                
! 
                Download the alpha5 release.
                
  
                Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
                
--- 103,111 ----
  
                Requirements
                
  
                  
! 
                • Python 2.2 or later (2.3 recommended)
                  • 
  
                • Should work on windows/unix/whatever...
                  • 
  
                
  
  
                Availability
                
! 
                Download the alpha6 release.
                
  
                Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
                

Index: download.ht
===================================================================
RCS file: /cvsroot/spambayes/website/download.ht,v
retrieving revision 1.18
retrieving revision 1.19
diff -C2 -d -r1.18 -r1.19
*** download.ht	8 Sep 2003 10:55:58 -0000	1.18
--- download.ht	19 Sep 2003 23:12:52 -0000	1.19
***************
*** 4,15 ****
  
  
                Source Releases
                
! 
                The fifth pre-release of version 1.0 of the SpamBayes project is available.
! Download version 1.0a5 from the sourceforge Files page as either a gzipped tarball or a zip file of the source files.
  
                
  
! 
                The primary goal of this pre-release is to shake out any packaging, installation, or integration issues that might be lurking. Feedback to spambayes@python.org. 
  
                Prerequisites:
  
                  Either: 
! 
                • Python 2.2.2, Python 2.3b1, or a CVS build of python, or
  
                • Python 2.2, 2.2.1, plus the latest email package.
  
                
--- 4,19 ----
  
  
                Source Releases
                
! 
                The sixth (and final alpha) pre-release of version 1.0 of the SpamBayes project is available.
! Download version 1.0a6 from the sourceforge Files page as either a gzipped tarball or a zip file of the source files.
  
                
  
! 
                The primary goal of this pre-release is to shake out any packaging, installation, or integration issues that might be lurking.
! In particular, the script names were changed for this release, and the options code was cleaned up a lot.
! Version 1.0 is now in 'feature freeze', so, while we will continue to resolve any bugs, no new features will be added.  This means
! that a beta, and then final, release for version 1.0 is not too far off.  New features will be added to the 1.1a1 release.
! Feedback to spambayes@python.org. 
  
                Prerequisites:
  
                  Either: 
! 
                • Python 2.2.2 or above, or a CVS build of python, or
  
                • Python 2.2, 2.2.1, plus the latest email package.
  
                
***************
*** 17,21 ****
  
                  
  
                • if you're running Outlook2000, change to the Outlook2000 directory and run 'addin.py'. See the 'about.html' file in the Outlook2000 directory for more.
! 
                • otherwise, consult the INTEGRATION.txt file in the package and choose the method that suits your setup best.
  
                
  
                
--- 21,25 ----
  
                  
  
                • if you're running Outlook2000, change to the Outlook2000 directory and run 'addin.py'. See the 'about.html' file in the Outlook2000 directory for more.
! 
                • otherwise, consult the README.txt file in the package and choose the method that suits your setup best.
  
                
  
                

Index: index.ht
===================================================================
RCS file: /cvsroot/spambayes/website/index.ht,v
retrieving revision 1.26
retrieving revision 1.27
diff -C2 -d -r1.26 -r1.27
*** index.ht	11 Sep 2003 07:58:34 -0000	1.26
--- index.ht	19 Sep 2003 23:12:52 -0000	1.27
***************
*** 6,10 ****
  
                News
                
  
                Outlook plugin release 008 is available. See the Windows page for more. Note that the Outlook plugin is now available from Sourceforge, on this project's Files page.
                
! 
                Fifth pre-release of spambayes available. See the download page for more.
                
  
                You may also like to see what other people have been saying about us in the press and elsewhere.
                
  
--- 6,10 ----
  
                News
                
  
                Outlook plugin release 008 is available. See the Windows page for more. Note that the Outlook plugin is now available from Sourceforge, on this project's Files page.
                
! 
                Sixth (and final alpha) pre-release of spambayes available. See the download page for more.
                
  
                You may also like to see what other people have been saying about us in the press and elsewhere.
                
  



From anadelonbrin at users.sourceforge.net  Fri Sep 19 19:27:21 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 19 19:27:23 2003
Subject: [Spambayes-checkins] 
	spambayes HAMMIE.txt, 1.5, NONE INTEGRATION.txt, 1.10, NONE
Message-ID: 

Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv10318

Removed Files:
	HAMMIE.txt INTEGRATION.txt 
Log Message:
All the information in here is now in README.txt, and is more up-to-date.

--- HAMMIE.txt DELETED ---

--- INTEGRATION.txt DELETED ---



From anadelonbrin at users.sourceforge.net  Fri Sep 19 19:38:12 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 19 19:38:16 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_server.py, 1.6,
	1.7 sb_smtpproxy.py, 1.3, NONE
Message-ID: 

Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv12134/scripts

Modified Files:
	sb_server.py 
Removed Files:
	sb_smtpproxy.py 
Log Message:
If anyone wants to use the smtp proxy, then they can do so via sb_server, with or
without using the pop3 proxy as well.  This means that sb_smtpproxy doesn't really
need to exist anymore, and the smtpproxy stuff would be better as a module.  Do this.

We had too many message classes!  As discussed (by me and Mark, mostly) a long
time back on spambayes-dev, start consolidating these (I was waiting for 1.1).

Add the various interface improvements discussed on spambayes-dev.  In particular,
an advanced 'find token' query is available, the 'find message' query is improved,
and the review messages page is more customisable.

Index: sb_server.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_server.py,v
retrieving revision 1.6
retrieving revision 1.7
diff -C2 -d -r1.6 -r1.7
*** sb_server.py	18 Sep 2003 03:58:59 -0000	1.6
--- sb_server.py	19 Sep 2003 23:38:10 -0000	1.7
***************
*** 477,481 ****
                          # Write the message into the Unknown cache.
                          message = state.unknownCorpus.makeMessage(msg.getId())
!                         message.setSubstance(msg.as_string())
                          state.unknownCorpus.addMessage(message)
  
--- 477,481 ----
                          # Write the message into the Unknown cache.
                          message = state.unknownCorpus.makeMessage(msg.getId())
!                         message.setPayload(msg.as_string())
                          state.unknownCorpus.addMessage(message)
  
***************
*** 493,497 ****
                  body = re.split(r'\n\r?\n', messageText, 1)[1]
                  messageText = "\r\n".join(headers) + "\r\n\r\n" + body
- 
              except:
                  # Something nasty happened while parsing or classifying -
--- 493,496 ----
***************
*** 751,758 ****
      # SMTP proxy information in their configuration, then nothing will
      # happen.
!     import sb_smtpproxy
!     servers, proxyPorts = sb_smtpproxy.LoadServerInfo()
!     proxyListeners.extend(sb_smtpproxy.CreateProxies(servers, proxyPorts,
!                                                      state))
  
      # setup info for the web interface
--- 750,757 ----
      # SMTP proxy information in their configuration, then nothing will
      # happen.
!     from spambayes import smtpproxy
!     servers, proxyPorts = smtpproxy.LoadServerInfo()
!     proxyListeners.extend(smtpproxy.CreateProxies(servers, proxyPorts,
!                                                   state))
  
      # setup info for the web interface

--- sb_smtpproxy.py DELETED ---



From anadelonbrin at users.sourceforge.net  Fri Sep 19 19:38:12 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 19 19:38:19 2003
Subject: [Spambayes-checkins] spambayes/spambayes/resources ui.html, 1.19,
	1.20 ui_html.py, 1.20, 1.21
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes/resources
In directory sc8-pr-cvs1:/tmp/cvs-serv12134/spambayes/resources

Modified Files:
	ui.html ui_html.py 
Log Message:
If anyone wants to use the smtp proxy, then they can do so via sb_server, with or
without using the pop3 proxy as well.  This means that sb_smtpproxy doesn't really
need to exist anymore, and the smtpproxy stuff would be better as a module.  Do this.

We had too many message classes!  As discussed (by me and Mark, mostly) a long
time back on spambayes-dev, start consolidating these (I was waiting for 1.1).

Add the various interface improvements discussed on spambayes-dev.  In particular,
an advanced 'find token' query is available, the 'find message' query is improved,
and the review messages page is more customisable.

Index: ui.html
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/resources/ui.html,v
retrieving revision 1.19
retrieving revision 1.20
diff -C2 -d -r1.19 -r1.20
*** ui.html	30 Aug 2003 21:37:10 -0000	1.19
--- ui.html	19 Sep 2003 23:38:10 -0000	1.20
***************
*** 167,171 ****
      here, to be trained on later.  Click one of the
      Discard / Defer / Ham / Spam headers to check all of the
!     buttons in that section in one go.
                
  
      
                
--- 167,173 ----
      here, to be trained on later.  Click one of the
      Discard / Defer / Ham / Spam headers to check all of the
!     buttons in that section in one go.  Click one of the other
!     headers to sort messages (within their classification) by that
!     header.
                
  
      
***************
*** 206,213 ****
      
      
!     
!         
!         
+     
+         
+     
                
          Messages classified as TYPE:From:To:
          
              Discard /
--- 208,226 ----
      
      
!     
                
!     
                
!     
!     
!         
!             
!         
!         
!             
!         
!         
!         
!         
+         
+             
+         
+     
!         
!             
!             
!         
!         
!         
--- 259,265 ----
                     id="spam"    value='spam'/>
          
!         
!         
***************
*** 268,284 ****
  
                wordQuery
                
! 
!     
!     
!     
- 
- 
  
                findMessage
                
  
!     
!     
!     
  
--- 294,366 ----
  
                wordQuery
                
!     
!     
                
          Messages classified as TYPE:
                
!                 
!                     Subject
!                 
!             Received:
          
              Discard /
***************
*** 217,228 ****
              
          
      
                
      
                
!             
!                 Re: Spambayes and PyMeld rock!  8-)
!         Richie Hindle <richie@entrian.com>Spambayes List <spambayes@python.org>
           
              
          Score:
      
                
      
                
!                 
!                     Richie Hindle <richie@entrian.com>
!                 
!             Sat, 11 Sep 2003 19:03:11
           
              
          Show clues
      
                
      
                   
                0.00%Clues |
!         Tokens
      
                
      
                   
                
  
      
  
                
      
  
                
!     
!     
!     
!         
!     
!     
!     
                
!         
!         
                
!         
!                     		
!             
!             Basic Query
!             
                
!             
!             Wildcard Query
!             
                
!             
!             Regular Expression Query
!             
                
!             
!             Ignore Case
!             
                
!             Maximum results
!             
!             
                
!         
                
      
  
                
  
                findMessage
                
  
!     
                
!     
!     
!     
!     
!     
!     
!     
                
!         Search in...
!         
                
!         
!         SpamBayes ID
!         
                
!         
!         Subject
!         
                
!         
!         Message headers
!         
                
!         
!         Message body
!                     		
!         
!         Ignore Case
!         
                
!         Maximum results
!         
!         
                
!                     		
!         
!         
!     
                
      
                
  
***************
*** 293,296 ****
--- 375,394 ----
      0.789.
                
      
+     
+     
+     
+         
+         
+         
+         
+     
+     
+         
+         
+         
+         
+     
+     
                Word# Spam# HamProbability
                spambayes123436.789
                
+                     
  
  
***************
*** 299,309 ****
  
      
!     
                Spam probability: 0.123
                
      
          (The table of clues goes here, like this but in a headedBox):
          
!                 
                
              
                
                  Example word0.123
              
                
          
                
--- 397,416 ----
  
      
!     
                Spam probability: 0.123.
!     Original probability: 0.125.
                
      
          (The table of clues goes here, like this but in a headedBox):
          
+             
+                 
+                 
+                 
+                 
+             
!                 
!                 
!                 
                WordProbabilityTimes in hamTimes in spam
                
              
                
                  Example word0.12312
              
                
          
                
***************
*** 318,322 ****
  
      
!     
                
  
      
                This page allows you to change the options that control how
--- 425,429 ----
  
      
!     
  
      
                This page allows you to change the options that control how
***************
*** 379,389 ****
      
  
!     
  
!     
                
      
                
        
      
                
!     
                
  
      
                
--- 486,496 ----
      
  
!     
                
  
!     
                
      
                
        
      
                
!     
                
  
      
                

Index: ui_html.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/resources/ui_html.py,v
retrieving revision 1.20
retrieving revision 1.21
diff -C2 -d -r1.20 -r1.21
*** ui_html.py	30 Aug 2003 21:37:10 -0000	1.20
--- ui_html.py	19 Sep 2003 23:38:10 -0000	1.21
***************
*** 6,87 ****
  
  import zlib
! data = zlib.decompress("x;sƱg&\005\035\033T\"!v\023Ԗ8ٱ*1d2\031\0018\000\034\017I{wÇ$\
! :if!=ο:x✽[~x.~|S\034L;>Nϖg;d˂ge\\\
! 2tz|S\011}\013\036w\025W`qpr\006|+Jc)\012}VbC1\000MU\
! \007\037u|p.\012NB\011jd F8l::Tu<>\013QZ3z&g?yf\
! }a\007\000,\020<#^A\003t,\014fe%\032\0042ڲOl\005\020f\033\031E̓}V\
! e6\"\025U\034\023b\035g3}E\003\016pJ\026)\000\013>K\006Cb_H|v\
! \033Áލ\023m^{/\012&\024k\016D\000L\"\032q\033;Zy0$\0036i5Z\000\000w(JlC=\
! qՃ[{\00451`sq-3u\035\020u\016194п&'dOO\
! G\"\"j[|p\015\011?[\026&,\027 =e\016\032\016gn\0008]\010\027j@a˃2\
! (\021\005e~\002x\000\0164:\005PwQ\035V?c\022&\002\"\0205+Le*H\034)\012\026w\
! \007)\037_\016d-\023V\023S]Ի/@0EQ\034Hy*qv\016\035-ފK{1g8\
! \001r\023l\025'硌`Ao,\0228\003\"T\033\022)o\016x\026\035H\024\010\032flhdZ[\022^\002\
! %4߂u\0001`3Ȇ\"\011\0003!\012\036\" 8OSC@\024uV)h,ɢ-\020\024,\
! Il:MA\020jë.^jBU:g\0070J \037t@\"J\030TP̀%lNQbA$\
! {\020JMD\026+\017*VD\003#A\021t\014\016\007v\017\023PsR~\003\024z,\030D8\007\
! [\020feBD%&\020|z\033T\031#pD@\006qQv6D4v\000WS}26CAp\
! j<\023d#\024w!k\017\032QQ\\\005\036,BQ\000\013PM\032&L]Z\006q\025C;a6\027\001\
! 8qI-n^[83բ՞\031C.V\026Z\026c[\0275\037p+A,\002K[\015\0122,x\005,Vhn}EC\033z$TM\0028\011P\
! hU\006j\035\007,D:\010\010Q\012\003GvJm\020kPٺƍV$Q\004Vt@\
! ZZYR4 %Ц2m_y\025\011hK\034Ă\021\022@Չ\003bT\012\012\024\024Q=>\
! '(zrtcIڭ\"I`fQꬪ\032pf\027\007^GCҌ\
! \024\025\024\001\001D7\006F8u]\010r\013!r{\001uU\004\002=\015T\007^\001*xU\023_z.1|\
! 8xZzӰ˚An\031\032If\021\036,q4\024pO\013\007z I(\012'+\001\
! iXWV0P\031lWD\025o%\0231L\0122\007\010\037+zk\017i݁s\011uV&\
! \\u.\001\025jy7:w\031l}W\"\031Dq\031qQ'O\031j\022<{S/v\
! W<\035O\003>^\030KLv/[4\020\025?\000M,[\033`[\005I@Pt'/\
! &]\011Ja8Oz\013\014!+m]ُ-SɊ'҉GN@\024\027ū\030֎fuQCoA\006O\
! \015gmcVօM%҃\01130b\030>vzk ]9S6\031Rs?\
! Žٛcq{&\025\000\014׏f!;\035\025U\005.\015c\000o\015\0338\011b\020\023\014\032h\
! \036v\001YnJᵣc\026S\014jE?@\036\032\034&k\031F\023?iU\"k\011~gq!BN\
! \016@=+S`*\020CQj\015Mi`#?\022$\0273Z\033Y ̆f-\024bIx\021\021M\
! ;zJzHq_I\015 6\004S-`PUNC\013+[Sś\015Bk0ا\033\021x\016Ą(\002cӠ*\
! \012\034k)\000\002\0250\0254w]\004`X…p&~_\015VAc8L\020\
! P2EĦ7\021D&`&\0110CI#j'r>8\003FuYˮL\002NC3=\000S\003\
! \031:E@a\010\035\013T@\006t\012\\\027\000Ȁ|4:]#aއ\036\
! t0( +ZZ%\"cn] d\015\034\0078K\017@\034Z\031ԎۣT.XȁoR\017\
! !w\0170s7{C\026\"\010870~w~UO~>={|\016\
! LI̔\016z3~\006ARTʧ\026x^D2Sp'Gɞ?gݖI\"5D\035S?uHRM\
! 0\"\011ٜ\005\0164<\022b\006J\003\020s\004OzB\035/ܙ7R63͎\026\
! \013vm:\000_lDA85uRgPwRN\024`v^\015gz\034 );О~Q=dG5\
! K\033\036\021\015/>^-]\037\033&ZW]UAv,U\035\030\033\011~Q?\030Co9߼dX֟/9\010\
! qַLGpzs1\000ЯT@Ȼ\021\020ȸb:kM`R\003Fu|\
! LNA#\003M7@\002#Lp7@A#N`[R,\031\031nbKyAL\035?,St'C\
! nMu\011]{sp\017%R,\024\013\034*=\012\031|\007{bx\"\007s׎\031Xwq\026Daⶠ?\
! \004,x[cG\000eqY\02149D\026\007&ĬM\016P\003ץQ,\005Ւ\020\
! O\006\024Q\0247l\0339=@1pZ{N]\0240\004]\012x\"@a\000w\003!hoГJf\
! ̪Fhw\012kv!\030qQ\016\022ݠ$TCH\024e\000i\007\011\003w=I\035\031iӺao\
! suH\036\015\003\033s\032N\024F3\001n\"/*yQ\021V\007\021Q1U4\024hM\032TY\\X\
! n\031~c&\012ҭ\025S=kr\002%XPiHP@wI\023u\001QqE=\012BgvV\030\
! }e^w`!U`b\025\016GW*Ŋ\00381\006Ty)_m\010o=\012\
! ЧA-8ÑM\002\005c\023]c2\005@b)\0111\002W\026\031W2\007V\036*\004D[C`H-\
! h\007V/(tM&wHBs\014eH+rUM=%96_u\\\
! jHnW4hx|\000g7Ǿ\
! a;U\000\032ݍ\037\033ξp1\015Ǯz\005s\016ß/ɥzq2Fl\
! $H&<\036\016_u(1\026_r\005\022Z'ŹÀ\007\013ao\013ro\036(\023c(\017\030\010\037l\006\024{\031Y\
! \003~\020\005X/NY\031\015pj޼\001)ĒGɅrgGKhVz>[o\021\
! Q0gz\002Rr6b\000@¯\003\026Y\022`<\011\017gBĊћ\022W<\
! Ux+2@\001c\023x\000/\\\021H#\034{\032\013aM\004,\023\"\001\0328\014\014Di\\Ɯ\036\024DV&E\004\
! Ē,|\002J\022OLͦ NvE/_1r2eJ?`@,B\030VU@.Ss\
! \020Ni齈wZ,R[\002?\010\0060\020\003\"`\003\007$MS\030T%\007@6^*\
! <\002\035(\032o`-WڬI\022!\034u3\025J}Z{\013\024*ц@|Fa\003T8(~*\026\000\";>8\
! \033AB0k|B\024+j\017\032VaTL#\010B\021m^ziB\007JYeAP\
! 7qX\012~\034\024k!;#\031jת$Ij\023ig|2qCzJ?\
! *KA\015qߊWE@SuGU|\020_x\017Ά\006l\0162+,vΙLEh0p\
! \016\025>,d<ߑR(\003 T30@\002\022\036\022ݕL\001 A=fqq?\024o2!\0020w\
! \027b@@Fb\010C\014\013m`gC$֮>E8QFW&߂v\\r]䉫\027\021\006\
! ;8|*\005\027F\037\034\007J\003&35\000%5*jL.Y\036d\000<\024[\006!\004*\
! \007p4\001?}2\012\032\005'\034\034\013!\003V`\010ގ@\\7\032@8(8\"\"\000b\"\
! V5ܶtH(V\\Tܠ&pM\011?\001-BZ+\013ZH\003\023\033FY/\010\030\012d\
! &P#>ǧ(|jd\005bcI:\005\"gY[cVg\020\032ri׏]\036\034\
! =\030f\004E\010^\004t:\024\010\014\002u#(,Dm\000U\031\0100P\037DͼExSw\
! <\\X0k\023tZ1\036ijAVAa\031.R͆w\007\
! U30[esr\017\035PDͽIi`Ợ$\036.!F\017nj^b \
! Φ~5-F#Ol8\017酶\004fIb٬W\021\026yy\012A(n\023\
! \024f>n=&N\002Rh6σ\000]\012s{0,x(b%Nh\026ᲵIĬ\
! 5 nW^~J2\023n%=l\0139A}0$E\014ӧ$5\006rޤshؤn\013\002\021\
! tk?/ţAc=LU\012\000\003EMl*\004X6r\031;Lf]\036?\\!\002\
! &$0{$]HOy!:Pi\031\013%\0065_\"P%AgflZh[V\025&\020w6\003\027R\
! dhpҨ\001(أr^7\010]\001n(rʹ)35|\026GbEk!YVbow\
! \036uDʪ֦\035#%-(*?VR\015HB\026XPUMC;+OZRX\021+uy\032\026싅\010I42!\
! tZ\026\005b\006\022\000\021?\012X;A.d\020b\0021RDB\037tyG+Fuŕ@\0154\
! 1\035&^h\035QNQ\036,dCF7\010D#`\0114\003Uo%I\017@\011]g\\\025\033\001\
! r}+(Y5*\000e\037\005\032ΦьiK\023/1\001q>\024x3I\"(?s\027Q\030B\\Űw\
! tI7FReng\003\006\002\011x꯮zG9\021טnhn:\015\016r\012\\Zr*\
! Osw%d\034\0020ʑ? 1JJa.\\o\003,qp}x݄\021Z~떂v*э\
! -\033t\015IA\026F,\027\014\037z7^\\Ue\\N&\012'mA6>f'\
! P\006\022\001Y \026\034T\035]?6H^vN3\027ACE쌺\001_\
! l;U9$'T\007\"\001^cwg\010\005z\0321[\000~!j\015\031SW\
! +\016\012NDv\001~\004\011U\001a\024=2Ⱥ՗pou\036FƲ#\032^~;\011sM\
! Y@6\000Ux\021o˩YDpCp7\013\025Q1ůPY]S\005jS;\
! U\013Sn:ONX0&\006!\032jDߍ蕘v\033_Z)\035\004\002\022n\
! VlD\033=+Ze'r\024\026\011\007N\015\032\\\013\000\"#\025\000\0148Xq\
! y(\"Zx\005 \":\013`,Fd\036BV]`\010 },H\031N6H_TU[\005\0227r\
! U6xo˷МUB3jX\030ܛ(XD]EI\010 E;}\020'XR~\
! \017glvqSZW\026&1\010~@j\0131I#`Z'\030[\036n\035\037\033ƾ;]HT\
! a\0211/\016hnEʎ<\037{Ѩ\010\001\001\02693S\004\031\017#i@ax&8\0253Yv`\
! ObE\035Ū\012؍\006ѱ\016g\017V&um'F?\021ڊ\022\014ꋠN\005PM\017 T\
! G9V;jj\0333V1-xW\015&X.\031\035\0046[\007g] K$ctt\
! RMjҢ\"t\032Bיi\024G׮3J:**SȠI!یCV1u|o_3`Ϸ6d\
! {6eʣ\001\011]s7Wr!8!8dd,8oz\021:S1$e\022D\
! 5s}z\0124x\0041^'d)\\Ҷ\015uUƻKEà]zIo\
! \014dywurc=C\037!\005U*\002N4S\001.!Y{Ur\010Pf9Xvʣ\025Z_:{P<\024\
! J}\011G':q\003ۼ0_W~cJg\000yX:hNxf\025)NAt_\036t\003\021}\"\
! \"C\016C}}Gd.\017dwo6\037\013|\0340{AD\0139jD-Km\016yc\037OXW\
! a;e6~`2lasEoǎ^\006\020oR- :[i\032ڃC'q\012\035\
! ԛ<ϰ%\016n\036OZB\025tb(Pj/\\SjQcL{\0354\
! $\001=&U&n\017P.\025\032:Ĕ\016\0270[}]W\002:jۖ5\0372D_\
! Yt9\021͐jd\"RC\"O|W]\034\015mH\020>84lLƀ\
! vN\010s\015\010t1EZ\027L\022Y\\ۣ|jK<}bHkk7\010RR|\
! W\021GT|l\022\027j\007刊nk#Xʄ;Č*Y;b©\033\020?P\
! #[ͨS3vlDީ(0W3Jg/\024eK%ۘO7\027e\021Uҷ㠾*jP\
! \0043\017Rӧ\017Q!OИ޿sQf\"\007\036`ֹX{w2lZ\
! r.\030(2h6X#@\027\0254Bn\022\002f\017\003_\035{lt2N\
! GxtH32h,t<\034VAU@\032l>\030:\023KCSI)\033V\015?0\033\034\
! rh3ci\0344pLg<\013\011eHZ\010v&'G__Z^")
  ### end



From anadelonbrin at users.sourceforge.net  Fri Sep 19 19:47:13 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 19 19:48:39 2003
Subject: [Spambayes-checkins] spambayes/spambayes ProxyUI.py,1.24,1.25
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv13513/spambayes

Modified Files:
	ProxyUI.py 
Log Message:
Uncommitted Skip's rows_per_section improvement by mistake.  Return it.

Index: ProxyUI.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/ProxyUI.py,v
retrieving revision 1.24
retrieving revision 1.25
diff -C2 -d -r1.24 -r1.25
*** ProxyUI.py	19 Sep 2003 23:38:10 -0000	1.24
--- ProxyUI.py	19 Sep 2003 23:47:11 -0000	1.25
***************
*** 283,287 ****
  
          keyedMessageInfo = self._sortMessages(keyedMessageInfo, sort_order)
!         for key, messageInfo in keyedMessageInfo:
              unused, unused, messageInfo.received = \
                      self._getTimeRange(self._keyToTimestamp(key))
--- 283,288 ----
  
          keyedMessageInfo = self._sortMessages(keyedMessageInfo, sort_order)
!         nrows = options["html_ui", "rows_per_section"]
!         for key, messageInfo in keyedMessageInfo[:nrows]:
              unused, unused, messageInfo.received = \
                      self._getTimeRange(self._keyToTimestamp(key))



From anadelonbrin at users.sourceforge.net  Fri Sep 19 19:38:12 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 19 20:38:41 2003
Subject: [Spambayes-checkins] spambayes/spambayes smtpproxy.py, NONE,
	1.1 Corpus.py, 1.7, 1.8 FileCorpus.py, 1.6, 1.7 ImapUI.py,
	1.18, 1.19 Options.py, 1.79, 1.80 ProxyUI.py, 1.23,
	1.24 UserInterface.py, 1.24, 1.25 mboxutils.py, 1.2,
	1.3 message.py, 1.37, 1.38
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv12134/spambayes

Modified Files:
	Corpus.py FileCorpus.py ImapUI.py Options.py ProxyUI.py 
	UserInterface.py mboxutils.py message.py 
Added Files:
	smtpproxy.py 
Log Message:
If anyone wants to use the smtp proxy, then they can do so via sb_server, with or
without using the pop3 proxy as well.  This means that sb_smtpproxy doesn't really
need to exist anymore, and the smtpproxy stuff would be better as a module.  Do this.

We had too many message classes!  As discussed (by me and Mark, mostly) a long
time back on spambayes-dev, start consolidating these (I was waiting for 1.1).

Add the various interface improvements discussed on spambayes-dev.  In particular,
an advanced 'find token' query is available, the 'find message' query is improved,
and the review messages page is more customisable.

--- NEW FILE: smtpproxy.py ---
#!/usr/bin/env python

"""A SMTP proxy to train a Spambayes database.

You point SMTP Proxy at your SMTP server(s) and configure your email
client(s) to send mail through the proxy (i.e. usually this means you use
localhost as the outgoing server).

To setup, enter appropriate values in your Spambayes configuration file in
the "SMTP Proxy" section (in particular: "remote_servers", "listen_ports",
and "use_cached_message").  This configuration can also be carried out via
the web user interface offered by POP3 Proxy and IMAP Filter.

To use, simply forward/bounce mail that you wish to train to the
appropriate address (defaults to spambayes_spam@localhost and
spambayes_ham@localhost).  All other mail is sent normally.
(Note that IMAP Filter and POP3 Proxy users should not execute this script;
launching of SMTP Proxy will be taken care of by those applicatons).

There are two main forms of operation.  With both, mail to two
(user-configurable) email addresses is intercepted by the proxy (and is
*not* sent to the SMTP server) and used as training data for a Spambayes
database.  All other mail is simply relayed to the SMTP server.

If the "use_cached_message" option is False, the proxy uses the message
sent as training data.  This option is suitable for those not using
POP3 Proxy or IMAP Filter, or for those that are confident that their
mailer will forward/bounce messages in an unaltered form.

If the "use_cached_message" option is True, the proxy examines the message
for a unique spambayes identification number.  It then tries to find this
message in the pop3proxy caches and on the imap servers.  It then retrieves
the message from the cache/server and uses *this* as the training data.
This method is suitable for those using POP3 Proxy and/or IMAP Filter, and
avoids any potential problems with the mailer altering messages before
forwarding/bouncing them.

To use, enter the required SMTP server data in your configuration file and
run sb_server.py
"""

# This module is part of the spambayes project, which is Copyright 2002-3
# The Python Software Foundation and is covered by the Python Software
# Foundation license.

__author__ = "Tony Meyer "
__credits__ = "Tim Stone, all the Spambayes folk."

try:
    True, False
except NameError:
    # Maintain compatibility with Python 2.2
    True, False = 1, 0


todo = """
 o It would be nice if spam/ham could be bulk forwarded to the proxy,
   rather than one by one.  This would require separating the different
   messages and extracting the correct ids.  Simply changing to find
   *all* the ids in a message, rather than stopping after one *might*
   work, but I don't really know.  Richie Hindle suggested something along
   these lines back in September '02.
   
 o Suggestions?

Testing:

 o Test with as many clients as possible to check that the
   id is correctly extracted from the forwarded/bounced message.

MUA information:
A '*' in the Header column signifies that the smtpproxy can extract
the id from the headers only.  A '*' in the Body column signifies that
the smtpproxy can extract the id from the body of the message, if it
is there.
                                                        Header	Body
*** Windows 2000 MUAs ***
Eudora 5.2 Forward                                         *     *
Eudora 5.2 Redirect                                              *
Netscape Messenger (4.7) Forward (inline)                  *     *
Netscape Messenger (4.7) Forward (quoted) Plain      	         *
Netscape Messenger (4.7) Forward (quoted) HTML      	         *
Netscape Messenger (4.7) Forward (quoted) Plain & HTML       	 *       
Netscape Messenger (4.7) Forward (attachment) Plain 	   *     *	 
Netscape Messenger (4.7) Forward (attachment) HTML  	   *	 *
Netscape Messenger (4.7) Forward (attachment) Plain & HTML *  	 *
Outlook Express 6 Forward HTML (Base64)                          *
Outlook Express 6 Forward HTML (None)                            *
Outlook Express 6 Forward HTML (QP)                              *
Outlook Express 6 Forward Plain (Base64)                         *
Outlook Express 6 Forward Plain (None)                           *
Outlook Express 6 Forward Plain (QP)                             *
Outlook Express 6 Forward Plain (uuencoded)                      *
http://www.endymion.com/products/mailman Forward	             *
M2 (Opera Mailer 7.01) Forward                                   *
M2 (Opera Mailer 7.01) Redirect                            *     *
The Bat! 1.62i Forward (RFC Headers not visible)                 *
The Bat! 1.62i Forward (RFC Headers visible)               *     *
The Bat! 1.62i Redirect                                          *
The Bat! 1.62i Alternative Forward                         *     *
The Bat! 1.62i Custom Template                             *     *
AllegroMail 2.5.0.2 Forward                                      *
AllegroMail 2.5.0.2 Redirect                                     *
PocoMail 2.6.3 Bounce                                            *
PocoMail 2.6.3 Bounce                                            *
Pegasus Mail 4.02 Forward (all headers option set)         *     *
Pegasus Mail 4.02 Forward (all headers option not set)           *
Calypso 3 Forward                                                *
Calypso 3 Redirect                                         *     *
Becky! 2.05.10 Forward                                           *
Becky! 2.05.10 Redirect                                          *
Becky! 2.05.10 Redirect as attachment                      *     *
Mozilla Mail 1.2.1 Forward (attachment)                    *     *
Mozilla Mail 1.2.1 Forward (inline, plain)                 *1    *
Mozilla Mail 1.2.1 Forward (inline, plain & html)          *1    *
Mozilla Mail 1.2.1 Forward (inline, html)                  *1    *

*1 The header method will only work if auto-include original message
is set, and if view all headers is true.
"""

import string
import re
import socket
import asyncore
import asynchat
import getopt
import sys
import os

from spambayes import Dibbler
from spambayes import storage
from spambayes.message import sbheadermessage_from_string
from spambayes.tokenizer import textparts
from spambayes.tokenizer import try_to_repair_damaged_base64
from spambayes.Options import options
from sb_server import _addressPortStr, ServerLineReader
from sb_server import _addressAndPort

class SMTPProxyBase(Dibbler.BrighterAsyncChat):
    """An async dispatcher that understands SMTP and proxies to a SMTP
    server, calling `self.onTransaction(command, args)` for each
    transaction.

    self.onTransaction() should return the command to pass to
    the proxied server - the command can be the verbatim command or a
    processed version of it.  The special command 'KILL' kills it (passing
    a 'QUIT' command to the server).
    """

    def __init__(self, clientSocket, serverName, serverPort):
        Dibbler.BrighterAsyncChat.__init__(self, clientSocket)
        self.request = ''
        self.set_terminator('\r\n')
        self.command = ''           # The SMTP command being processed...
        self.args = ''              # ...and its arguments
        self.isClosing = False      # Has the server closed the socket?
        self.inData = False
        self.data = ""
        self.blockData = False
        self.serverSocket = ServerLineReader(serverName, serverPort,
                                             self.onServerLine)

    def onTransaction(self, command, args):
        """Overide this.  Takes the raw command and returns the (possibly
        processed) command to pass to the email client."""
        raise NotImplementedError

    def onProcessData(self, data):
        """Overide this.  Takes the raw data and returns the (possibly
        processed) data to pass back to the email client."""
        raise NotImplementedError

    def onServerLine(self, line):
        """A line of response has been received from the SMTP server."""
        # Has the server closed its end of the socket?
        if not line:
            self.isClosing = True

        # We don't process the return, just echo the response.
        self.push(line)
        self.onResponse()

    def collect_incoming_data(self, data):
        """Asynchat override."""
        self.request = self.request + data

    def found_terminator(self):
        """Asynchat override."""
        verb = self.request.strip().upper()
        if verb == 'KILL':
            self.socket.shutdown(2)
            self.close()
            raise SystemExit

        if self.request.strip() == '':
            # Someone just hit the Enter key.
            self.command = self.args = ''
        else:
            # A proper command.
            if self.request[:10].upper() == "MAIL FROM:":
                splitCommand = self.request.split(":", 1)
            elif self.request[:8].upper() == "RCPT TO:":
                splitCommand = self.request.split(":", 1)
            else:
                splitCommand = self.request.strip().split(None, 1)
            self.command = splitCommand[0]
            self.args = splitCommand[1:]

        if self.inData == True:
            self.data += self.request + '\r\n'
            if self.request == ".":
                self.inData = False
                cooked = self.onProcessData(self.data)
                self.data = ""
                if self.blockData == False:
                    self.serverSocket.push(cooked)
                else:
                    self.push("250 OK\r\n")
        else:
            cooked = self.onTransaction(self.command, self.args)
            if cooked is not None:
                self.serverSocket.push(cooked + '\r\n')
        self.command = self.args = self.request = ''

    def onResponse(self):
        # If onServerLine() decided that the server has closed its
        # socket, close this one when the response has been sent.
        if self.isClosing:
            self.close_when_done()

        # Reset.
        self.command = ''
        self.args = ''
        self.isClosing = False


class BayesSMTPProxyListener(Dibbler.Listener):
    """Listens for incoming email client connections and spins off
    BayesSMTPProxy objects to serve them."""

    def __init__(self, serverName, serverPort, proxyPort, trainer):
        proxyArgs = (serverName, serverPort, trainer)
        Dibbler.Listener.__init__(self, proxyPort, BayesSMTPProxy,
                                  proxyArgs)
        print 'SMTP Listener on port %s is proxying %s:%d' % \
               (_addressPortStr(proxyPort), serverName, serverPort)


class BayesSMTPProxy(SMTPProxyBase):
    """Proxies between an email client and a SMTP server, inserting
    judgement headers.  It acts on the following SMTP commands:

    o RCPT TO:
        o Checks if the recipient address matches the key ham or spam
          addresses, and if so notes this and does not forward a command to
          the proxied server.  In all other cases simply passes on the
          verbatim command.

     o DATA:
        o Notes that we are in the data section.  If (from the RCPT TO
          information) we are receiving a ham/spam message to train on,
          then do not forward the command on.  Otherwise forward verbatim.

    Any other commands are merely passed on verbatim to the server.          
    """

    def __init__(self, clientSocket, serverName, serverPort, trainer):
        SMTPProxyBase.__init__(self, clientSocket, serverName, serverPort)
        self.handlers = {'RCPT TO': self.onRcptTo, 'DATA': self.onData,
                         'MAIL FROM': self.onMailFrom}
        self.trainer = trainer
        self.isClosed = False
        self.train_as_ham = False
        self.train_as_spam = False

    def send(self, data):
        try:
            return SMTPProxyBase.send(self, data)
        except socket.error:
            # The email client has closed the connection - 40tude Dialog
            # does this immediately after issuing a QUIT command,
            # without waiting for the response.
            self.close()

    def close(self):
        # This can be called multiple times by async.
        if not self.isClosed:
            self.isClosed = True
            SMTPProxyBase.close(self)

    def stripAddress(self, address):
        """
        Strip the leading & trailing <> from an address.  Handy for
        getting FROM: addresses.
        """
        if '<' in address:
            start = string.index(address, '<') + 1
            end = string.index(address, '>')
            return address[start:end]
        else:
            return address

    def onTransaction(self, command, args):
        handler = self.handlers.get(command.upper(), self.onUnknown)
        return handler(command, args)

    def onProcessData(self, data):
        if self.train_as_spam:
            self.trainer.train(data, True)
            self.train_as_spam = False
            return ""
        elif self.train_as_ham:
            self.trainer.train(data, False)
            self.train_as_ham = False
            return ""
        return data

    def onRcptTo(self, command, args):
        toFull = self.stripAddress(args[0])
        if toFull == options["smtpproxy", "spam_address"]:
            self.train_as_spam = True
            self.train_as_ham = False
            self.blockData = True
            self.push("250 OK\r\n")
            return None
        elif toFull == options["smtpproxy", "ham_address"]:
            self.train_as_ham = True
            self.train_as_spam = False
            self.blockData = True
            self.push("250 OK\r\n")
            return None
        else:
            self.blockData = False
        return "%s:%s" % (command, ' '.join(args))
        
    def onData(self, command, args):
        self.inData = True
        if self.train_as_ham == True or self.train_as_spam == True:
            self.push("250 OK\r\n")
            return None
        rv = command
        for arg in args:
            rv += ' ' + arg
        return rv

    def onMailFrom(self, command, args):
        """Just like the default handler, but has the necessary colon."""
        rv = "%s:%s" % (command, ' '.join(args))
        return rv

    def onUnknown(self, command, args):
        """Default handler."""
        return self.request


class SMTPTrainer(object):
    def __init__(self, classifier, state=None, imap=None):
        self.classifier = classifier
        self.state = state
        self.imap = imap
    
    def extractSpambayesID(self, data):
        msg = message_from_string(data)

        # The nicest MUA is one that forwards the header intact.
        id = msg.get(options["Headers", "mailid_header_name"])
        if id is not None:
            return id

        # Some MUAs will put it in the body somewhere, while others will
        # put it in an attached MIME message.
        id = self._find_id_in_text(msg.as_string())
        if id is not None:
            return id

        # the message might be encoded
        for part in textparts(msg):
            # Decode, or take it as-is if decoding fails.
            try:
                text = part.get_payload(decode=True)
            except:
                text = part.get_payload(decode=False)
                if text is not None:
                    text = try_to_repair_damaged_base64(text)
            if text is not None:
                id = self._find_id_in_text(text)
                return id
        return None

    header_pattern = re.escape(options["Headers", "mailid_header_name"])
    # A MUA might enclose the id in a table, thus the convoluted re pattern
    # (Mozilla Mail does this with inline html)
    header_pattern += r":\s*(\\s*\\s*)?([\d\-]+)"
    header_re = re.compile(header_pattern)

    def _find_id_in_text(self, text):
        mo = self.header_re.search(text)
        if mo is None:
            return None
        return mo.group(2)

    def train(self, msg, isSpam):
        try:
            use_cached = options["smtpproxy", "use_cached_message"]
        except KeyError:
            use_cached = True
        if use_cached:
            id = self.extractSpambayesID(msg)
            if id is None:
                print "Could not extract id"
                return
            self.train_cached_message(id, isSpam)
        # Otherwise, train on the forwarded/bounced message.
        msg = sbheadermessage_from_string(msg)
        id = msg.setIdFromPayload()
        msg.delSBHeaders()
        if id is None:
            # No id, so we don't have any reliable method of remembering
            # information about this message, so we just assume that it
            # hasn't been trained before.  We could generate some sort of
            # checksum for the message and use that as an id (this would
            # mean that we didn't need to store the id with the message)
            # but that might be a little unreliable.
            self.classifier.learn(msg.asTokens(), isSpam)
        else:
            if msg.GetTrained() == (not isSpam):
                self.classifier.unlearn(msg.asTokens(), not isSpam)
                msg.RememberTrained(None)
            if msg.GetTrained() is None:
                self.classifier.learn(msg.asTokens(), isSpam)
                msg.RememberTrained(isSpam)

    def train_cached_message(self, id, isSpam):
        if not self.train_message_in_pop3proxy_cache(id, isSpam) and \
           not self.train_message_on_imap_server(id, isSpam):
            print "Could not find message (%s); perhaps it was " + \
                  "deleted from the POP3Proxy cache or the IMAP " + \
                  "server.  This means that no training was done." % (id, )

    def train_message_in_pop3proxy_cache(self, id, isSpam):
        if self.state is None:
            return False
        sourceCorpus = None
        for corpus in [self.state.unknownCorpus, self.state.hamCorpus,
                       self.state.spamCorpus]:
            if corpus.get(id) is not None:
                sourceCorpus = corpus
                break
        if corpus is None:
            return False
        if isSpam == True:
            targetCorpus = self.state.spamCorpus
        else:
            targetCorpus = self.state.hamCorpus
        targetCorpus.takeMessage(id, sourceCorpus)
        self.classifier.store()

    def train_message_on_imap_server(self, id, isSpam):
        if self.imap is None:
            return False
        msg = self.imap.FindMessage(id)
        if msg is None:
            return False
        if msg.GetTrained() == (not isSpam):
            msg.get_substance()
            msg.delSBHeaders()
            self.classifier.unlearn(msg.asTokens(), not isSpam)
            msg.RememberTrained(None)
        if msg.GetTrained() is None:
            msg.get_substance()
            msg.delSBHeaders()
            self.classifier.learn(msg.asTokens(), isSpam)
            msg.RememberTrained(isSpam)

def LoadServerInfo():
    # Load the proxy settings
    servers = []
    proxyPorts = []
    if options["smtpproxy", "remote_servers"]:
        for server in options["smtpproxy", "remote_servers"]:
            server = server.strip()
            if server.find(':') > -1:
                server, port = server.split(':', 1)
            else:
                port = '25'
            servers.append((server, int(port)))
    if options["smtpproxy", "listen_ports"]:
        splitPorts = options["smtpproxy", "listen_ports"]
        proxyPorts = map(_addressAndPort, splitPorts)
    if len(servers) != len(proxyPorts):
        print "smtpproxy:remote_servers & smtpproxy:listen_ports are " + \
              "different lengths!"
        sys.exit()
    return servers, proxyPorts    

def CreateProxies(servers, proxyPorts, trainer):
    """Create BayesSMTPProxyListeners for all the given servers."""
    proxyListeners = []
    for (server, serverPort), proxyPort in zip(servers, proxyPorts):
        listener = BayesSMTPProxyListener(server, serverPort, proxyPort,
                                          trainer)
        proxyListeners.append(listener)
    return proxyListeners

Index: Corpus.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Corpus.py,v
retrieving revision 1.7
retrieving revision 1.8
diff -C2 -d -r1.7 -r1.8
*** Corpus.py	4 May 2003 03:05:33 -0000	1.7
--- Corpus.py	19 Sep 2003 23:38:10 -0000	1.8
***************
*** 6,10 ****
      Corpus - a collection of Messages
      ExpiryCorpus - a "young" Corpus
-     Message - a subject of Spambayes training
      MessageFactory - creates a Message
  
--- 6,9 ----
***************
*** 65,69 ****
  
      MessageFactory is a required factory class, because Corpus is
!     designed to do lazy initialization of messages and as an abstract
      class, must know how to create concrete instances of the correct
      class.
--- 64,68 ----
  
      MessageFactory is a required factory class, because Corpus is
!     designed to do lazy initialization of messages and, as an abstract
      class, must know how to create concrete instances of the correct
      class.
***************
*** 74,78 ****
      '''
  
! # This module is part of the spambayes project, which is Copyright 2002
  # The Python Software Foundation and is covered by the Python Software
  # Foundation license.
--- 73,77 ----
      '''
  
! # This module is part of the spambayes project, which is Copyright 2002-3
  # The Python Software Foundation and is covered by the Python Software
  # Foundation license.
***************
*** 115,119 ****
  
      def addObserver(self, observer):
!         '''Register an observer, which must implement
          onAddMessage, onRemoveMessage'''
  
--- 114,118 ----
  
      def addObserver(self, observer):
!         '''Register an observer, which should implement
          onAddMessage, onRemoveMessage'''
  
***************
*** 192,210 ****
          '''Move a Message from another corpus to this corpus'''
  
-         # XXX Hack: Calling msg.getSubstance() here ensures that the
-         # message substance is in memory.  If it isn't, when addMessage()
-         # calls message.store(), which calls message.getSubstance(), that
-         # will try to load the substance from the as-yet-unwritten new file.
          msg = fromcorpus[key]
!         msg.getSubstance()
          fromcorpus.removeMessage(msg)
          self.addMessage(msg)
  
      def get(self, key, default=None):
-         # the old version would never return the default,
-         # it would just create a new message, even if that
-         # message did not exist in the cache
-         # we need to check for the key in our msgs, but we can't check
-         # for None, because that signifies a non-cached message
          if self.msgs.get(key, "") is "":
              return default
--- 191,200 ----
          '''Move a Message from another corpus to this corpus'''
  
          msg = fromcorpus[key]
!         msg.load() # ensure that the substance has been loaded
          fromcorpus.removeMessage(msg)
          self.addMessage(msg)
  
      def get(self, key, default=None):
          if self.msgs.get(key, "") is "":
              return default
***************
*** 214,218 ****
      def __getitem__(self, key):
          '''Corpus is a dictionary'''
- 
          amsg = self.msgs.get(key)
  
--- 204,207 ----
***************
*** 225,234 ****
      def keys(self):
          '''Message keys in the Corpus'''
- 
          return self.msgs.keys()
  
      def __iter__(self):
          '''Corpus is iterable'''
- 
          for key in self.keys():
              try:
--- 214,221 ----
***************
*** 239,248 ****
      def __str__(self):
          '''Instance as a printable string'''
- 
          return self.__repr__()
  
      def __repr__(self):
          '''Instance as a representative string'''
- 
          raise NotImplementedError
  
--- 226,233 ----
***************
*** 261,265 ****
      def __init__(self, expireBefore):
          '''Constructor'''
- 
          self.expireBefore = expireBefore
  
--- 246,249 ----
***************
*** 274,424 ****
  
  
- class Message:
-     '''Abstract Message class'''
- 
-     def __init__(self):
-         '''Constructor()'''
- 
-         # The text of the message headers and body are held in attributes
-         # called 'hdrtxt' and 'payload', created on demand in __getattr__
-         # by calling load(), which should in turn call setSubstance().
-         # This means you don't need to remember to call load() before
-         # using these attributes.
- 
-     def __getattr__(self, attributeName):
-         '''On-demand loading of the message text.'''
- 
-         if attributeName in ('hdrtxt', 'payload'):
-             self.load()
-         try:
-             return self.__dict__[attributeName]
-         except KeyError:
-             raise AttributeError, attributeName
- 
-     def load(self):
-         '''Method to load headers and body'''
- 
-         raise NotImplementedError
- 
-     def store(self):
-         '''Method to persist a message'''
- 
-         raise NotImplementedError
- 
-     def remove(self):
-         '''Method to obliterate a message'''
- 
-         raise NotImplementedError
- 
-     def __repr__(self):
-         '''Instance as a representative string'''
- 
-         raise NotImplementedError
- 
-     def __str__(self):
-         '''Instance as a printable string'''
- 
-         return self.getSubstance()
- 
-     def name(self):
-         '''Message may have a unique human readable name'''
- 
-         return self.__repr__()
- 
-     def key(self):
-         '''The key for this instance'''
- 
-         raise NotImplementedError
- 
-     def setSubstance(self, sub):
-         '''set this message substance'''
- 
-         bodyRE = re.compile(r"\r?\n(\r?\n)(.*)", re.DOTALL+re.MULTILINE)
-         bmatch = bodyRE.search(sub)
-         if bmatch:
-             self.payload = bmatch.group(2)
-             self.hdrtxt = sub[:bmatch.start(2)]
-         else:
-             # malformed message - punt
-             self.payload = sub
-             self.hdrtxt = ""
- 
-     def getSubstance(self):
-         '''Return this message substance'''
- 
-         return self.hdrtxt + self.payload
- 
-     def setSpamprob(self, prob):
-         '''Score of the last spamprob calc, may not be persistent'''
- 
-         self.spamprob = prob
- 
-     def tokenize(self):
-         '''Returns substance as tokens'''
- 
-         return tokenizer.tokenize(self.getSubstance())
- 
-     def createTimeStamp(self):
-         '''Returns the create time of this message'''
-         # Should return a timestamp like time.time()
- 
-         raise NotImplementedError
- 
-     def getFrom(self):
-         '''Return a message From header content'''
- 
-         if self.hdrtxt:
-             match = re.search(r'^From:(.*)$', self.hdrtxt, re.MULTILINE)
-             return match.group(1)
-         else:
-             return None
- 
-     def getSubject(self):
-         '''Return a message Subject header contents'''
- 
-         if self.hdrtxt:
-             match = re.search(r'^Subject:(.*)$', self.hdrtxt, re.MULTILINE)
-             return match.group(1)
-         else:
-             return None
- 
-     def getDate(self):
-         '''Return a message Date header contents'''
- 
-         if self.hdrtxt:
-             match = re.search(r'^Date:(.*)$', self.hdrtxt, re.MULTILINE)
-             return match.group(1)
-         else:
-             return None
- 
-     def getHeadersList(self):
-         '''Return a list of message header tuples'''
- 
-         hdrregex = re.compile(r'^([A-Za-z0-9-_]*): ?(.*)$', re.MULTILINE)
-         data = re.sub(r'\r?\n\r?\s',' ',self.hdrtxt,re.MULTILINE)
-         match = hdrregex.findall(data)
- 
-         return match
- 
-     def getHeaders(self):
-         '''Return message headers as text'''
- 
-         return self.hdrtxt
- 
-     def getPayload(self):
-         '''Return the message body'''
- 
-         return self.payload
- 
-     def stripSBDHeader(self):
-         '''Removes the X-Spambayes-Disposition: header from the message'''
- 
-         # This is useful for training, where a spammer may be spoofing
-         # our header, to make sure that our header doesn't become an
-         # overweight clue to hamminess
- 
-         raise NotImplementedError
- 
- 
  class MessageFactory:
      '''Abstract Message Factory'''
--- 258,261 ----
***************
*** 430,434 ****
      def create(self, key):
          '''Create a message instance'''
- 
          raise NotImplementedError
  
--- 267,270 ----

Index: FileCorpus.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/FileCorpus.py,v
retrieving revision 1.6
retrieving revision 1.7
diff -C2 -d -r1.6 -r1.7
*** FileCorpus.py	16 Sep 2003 04:42:32 -0000	1.6
--- FileCorpus.py	19 Sep 2003 23:38:10 -0000	1.7
***************
*** 86,89 ****
--- 86,90 ----
  
  from spambayes import Corpus
+ from spambayes import message
  from spambayes import storage
  import sys, os, gzip, fnmatch, getopt, errno, time, stat
***************
*** 116,120 ****
          # to the corpus should for the moment be handled by a complete
          # retraining.
- 
          for filename in os.listdir(directory):
              if fnmatch.fnmatch(filename, filter):
--- 117,120 ----
***************
*** 123,134 ****
      def makeMessage(self, key):
          '''Ask our factory to make a Message'''
- 
          msg = self.factory.create(key, self.directory)
- 
          return msg
  
      def addMessage(self, message):
          '''Add a Message to this corpus'''
- 
          if not fnmatch.fnmatch(message.key(), self.filter):
              raise ValueError
--- 123,131 ----
***************
*** 145,149 ****
      def removeMessage(self, message):
          '''Remove a Message from this corpus'''
- 
          if options["globals", "verbose"]:
              print 'removing',message.key(),'from corpus'
--- 142,145 ----
***************
*** 187,206 ****
  
  
! class FileMessage(Corpus.Message):
      '''Message that persists as a file system artifact.'''
  
      def __init__(self,file_name, directory):
          '''Constructor(message file name, corpus directory name)'''
! 
!         Corpus.Message.__init__(self)
          self.file_name = file_name
          self.directory = directory
  
!         # No calling of self.load() here - that's done on demand by
!         # Message.__getattr__.
  
      def pathname(self):
          '''Derive the pathname of the message file'''
- 
          return os.path.join(self.directory, self.file_name)
  
--- 183,202 ----
  
  
! class FileMessage(message.SBHeaderMessage):
      '''Message that persists as a file system artifact.'''
  
      def __init__(self,file_name, directory):
          '''Constructor(message file name, corpus directory name)'''
!         message.SBHeaderMessage.__init__(self)
          self.file_name = file_name
          self.directory = directory
+         self.loaded = False
  
!     def as_string(self):
!         self.load() # ensure that the substance is loaded
!         return message.SBHeaderMessage.as_string(self)
  
      def pathname(self):
          '''Derive the pathname of the message file'''
          return os.path.join(self.directory, self.file_name)
  
***************
*** 215,218 ****
--- 211,217 ----
          # messages gzipped.  If someone can think of a classier (pun
          # intended) way of doing this, be my guest.
+         if self.loaded:
+             return
+ 
          if options["globals", "verbose"]:
              print 'loading', self.file_name
***************
*** 227,231 ****
          else:
              try:
!                 self.setSubstance(fp.read())
              except IOError, e:
                  if str(e) == 'Not a gzipped file':
--- 226,230 ----
          else:
              try:
!                 self.setPayload(fp.read())
              except IOError, e:
                  if str(e) == 'Not a gzipped file':
***************
*** 239,246 ****
                              raise
                      else:
!                         self.setSubstance(fp.read())
                          fp.close()
              else:
                  fp.close()
  
      def store(self):
--- 238,246 ----
                              raise
                      else:
!                         self.setPayload(fp.read())
                          fp.close()
              else:
                  fp.close()
+         self.loaded = True
  
      def store(self):
***************
*** 250,258 ****
              print 'storing', self.file_name
  
!         pn = self.pathname()
!         fp = open(pn, 'wb')
!         fp.write(self.getSubstance())
          fp.close()
  
      def remove(self):
          '''Message hara-kiri'''
--- 250,261 ----
              print 'storing', self.file_name
  
!         fp = open(self.pathname(), 'wb')
!         fp.write(self.as_string())
          fp.close()
  
+     def setPayload(self, payload):
+         self.loaded = True
+         message.SBHeaderMessage.setPayload(self, payload)
+ 
      def remove(self):
          '''Message hara-kiri'''
***************
*** 275,287 ****
  
          elip = ''
!         sub = self.getSubstance()
  
!         if options["globals", "verbose"]:
!             sub = self.getSubstance()
!         else:
              if len(sub) > 20:
-                 sub = sub[:20]
                  if len(sub) > 40:
!                     sub += '...' + sub[-20:]
  
          pn = os.path.join(self.directory, self.file_name)
--- 278,289 ----
  
          elip = ''
!         sub = self.as_string()
  
!         if not options["globals", "verbose"]:
              if len(sub) > 20:
                  if len(sub) > 40:
!                     sub = sub[:20] + '...' + sub[-20:]
!                 else:
!                     sub = sub[:20]
  
          pn = os.path.join(self.directory, self.file_name)
***************
*** 293,297 ****
      def __str__(self):
          '''Instance as a printable string'''
- 
          return self.__repr__()
  
--- 295,298 ----
***************
*** 330,334 ****
          pn = self.pathname()
          gz = gzip.open(pn, 'wb')
!         gz.write(self.getSubstance())
          gz.flush()
          gz.close()
--- 331,335 ----
          pn = self.pathname()
          gz = gzip.open(pn, 'wb')
!         gz.write(self.as_string())
          gz.flush()
          gz.close()
***************
*** 390,394 ****
  
      m1 = fmClass('XMG00001', 'fctestspamcorpus')
!     m1.setSubstance(testmsg2())
  
      print '\n\nAdd a message to hamcorpus that does not match the filter'
--- 391,395 ----
  
      m1 = fmClass('XMG00001', 'fctestspamcorpus')
!     m1.setPayload(testmsg2())
  
      print '\n\nAdd a message to hamcorpus that does not match the filter'
***************
*** 445,457 ****
      msg = spamcorpus['MSG00001']
      print msg
-     print '\n\nThis is some vital information in the message'
-     print 'Date header is',msg.getDate()
-     print 'Subject header is',msg.getSubject()
-     print 'From header is',msg.getFrom()
- 
-     print 'Header text is:',msg.getHeaders()
-     print 'Headers are:',msg.getHeadersList()
-     print 'Body is:',msg.getPayload()
- 
  
  
--- 446,449 ----
***************
*** 551,563 ****
  
      m1 = fmClass('MSG00001', 'fctestspamcorpus')
!     m1.setSubstance(tm1)
      m1.store()
  
      m2 = fmClass('MSG00002', 'fctestspamcorpus')
!     m2.setSubstance(tm2)
      m2.store()
  
      m3 = fmClass('MSG00003', 'fctestunsurecorpus')
!     m3.setSubstance(tm1)
      m3.store()
  
--- 543,555 ----
  
      m1 = fmClass('MSG00001', 'fctestspamcorpus')
!     m1.setPayload(tm1)
      m1.store()
  
      m2 = fmClass('MSG00002', 'fctestspamcorpus')
!     m2.setPayload(tm2)
      m2.store()
  
      m3 = fmClass('MSG00003', 'fctestunsurecorpus')
!     m3.setPayload(tm1)
      m3.store()
  
***************
*** 571,583 ****
  
      m4 = fmClass('MSG00004', 'fctestunsurecorpus')
!     m4.setSubstance(tm1)
      m4.store()
  
      m5 = fmClass('MSG00005', 'fctestunsurecorpus')
!     m5.setSubstance(tm2)
      m5.store()
  
      m6 = fmClass('MSG00006', 'fctestunsurecorpus')
!     m6.setSubstance(tm2)
      m6.store()
  
--- 563,575 ----
  
      m4 = fmClass('MSG00004', 'fctestunsurecorpus')
!     m4.setPayload(tm1)
      m4.store()
  
      m5 = fmClass('MSG00005', 'fctestunsurecorpus')
!     m5.setPayload(tm2)
      m5.store()
  
      m6 = fmClass('MSG00006', 'fctestunsurecorpus')
!     m6.setPayload(tm2)
      m6.store()
  
***************
*** 693,697 ****
  
  if __name__ == '__main__':
- 
      try:
          opts, args = getopt.getopt(sys.argv[1:], 'estgvhcu')
--- 685,688 ----

Index: ImapUI.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/ImapUI.py,v
retrieving revision 1.18
retrieving revision 1.19
diff -C2 -d -r1.18 -r1.19
*** ImapUI.py	18 Sep 2003 03:58:59 -0000	1.18
--- ImapUI.py	19 Sep 2003 23:38:10 -0000	1.19
***************
*** 103,106 ****
--- 103,108 ----
      ('Tokenizer',           'summarize_email_prefixes'),
      ('Tokenizer',           'summarize_email_suffixes'),
+     ('Interface Options',   None),
+     ('html_ui',             'display_adv_find'),
  )
  
***************
*** 134,143 ****
          filter
                and Configure folders to train"""
          content = (self._buildBox('Status and Configuration',
                                    'status.gif', statusTable % stateDict)+
                     self._buildTrainBox() +
                     self._buildClassifyBox() +
!                    self._buildBox('Word query', 'query.gif',
!                                   self.html.wordQuery)
                     )
          self._writePreamble("Home")
--- 136,148 ----
          filter
                and Configure folders to train"""
+         findBox = self._buildBox('Word query', 'query.gif',
+                                  self.html.wordQuery)
+         if not options["html_ui", "display_adv_find"]:
+             del findBox.advanced
          content = (self._buildBox('Status and Configuration',
                                    'status.gif', statusTable % stateDict)+
                     self._buildTrainBox() +
                     self._buildClassifyBox() +
!                    findBox
                     )
          self._writePreamble("Home")

Index: Options.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v
retrieving revision 1.79
retrieving revision 1.80
diff -C2 -d -r1.79 -r1.80
*** Options.py	18 Sep 2003 13:55:11 -0000	1.79
--- Options.py	19 Sep 2003 23:38:10 -0000	1.80
***************
*** 759,770 ****
       IP_LIST, RESTORE),
  
!     ("display_to", "Display To: in message review", False,
       """When reviewing messages via the web user interface, you are
!      presented with the message subject, the address the message is
!      from, and its classification.  If you set this option, you will
!      also be shown the address the message was to.  This might be
!      useful if you receive mail from multiple accounts, or if you
!      want to quickly identify mail received via a mailing list.""",
       BOOLEAN, RESTORE),
  
      ("http_authentication", "HTTP Authentication", "None",
--- 759,811 ----
       IP_LIST, RESTORE),
  
!     ("display_headers", "Headers to display in message review", ("Subject", "From"),
       """When reviewing messages via the web user interface, you are
!      presented with various information about the message.  By default, you
!      are shown the subject and who the message is from.  You can add other
!      message headers to display, however, such as the address the message
!      is to, or the date that the message was sent.""",
!      HEADER_NAME, RESTORE),
! 
!     ("display_received_time", "Display date received in message review", False,
!      """When reviewing messages via the web user interface, you are
!      presented with various information about the message.  If you set
!      this option, you will be shown the date that the message was received.
!      """,
       BOOLEAN, RESTORE),
+ 
+     ("display_score", "Display score in message review", False,
+      """When reviewing messages via the web user interface, you are
+      presented with various information about the message.  If you
+      set this option, this information will include the score that
+      the message received when it was classified.  You might wish to
+      see this purely out of curiousity, or you might wish to only
+      train on messages that score towards the boundaries of the
+      classification areas.  Note that in order to use this option,
+      you must also enable the option to include the score in the
+      message headers.""",
+      BOOLEAN, RESTORE),
+ 
+     ("display_adv_find", "Display the advanced find query", False,
+      """Present advanced options in the 'Word Query' box on the front page,
+      including wildcard and regular expression searching.""",
+      BOOLEAN, RESTORE),
+ 
+     ("default_ham_action", "Default training for ham", "ham",
+      """When presented with the review list in the web interface,
+      which button would you like checked by default when the message
+      is classified as ham?""",
+      ("ham", "spam", "discard", "defer"), RESTORE),
+ 
+     ("default_spam_action", "Default training for spam", "spam",
+      """When presented with the review list in the web interface,
+      which button would you like checked by default when the message
+      is classified as spam?""",
+      ("ham", "spam", "discard", "defer"), RESTORE),
+ 
+     ("default_unsure_action", "Default training for unsure", "defer",
+      """When presented with the review list in the web interface,
+      which button would you like checked by default when the message
+      is classified as unsure?""",
+      ("ham", "spam", "discard", "defer"), RESTORE),
  
      ("http_authentication", "HTTP Authentication", "None",

Index: ProxyUI.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/ProxyUI.py,v
retrieving revision 1.23
retrieving revision 1.24
diff -C2 -d -r1.23 -r1.24
*** ProxyUI.py	18 Sep 2003 13:55:11 -0000	1.23
--- ProxyUI.py	19 Sep 2003 23:38:10 -0000	1.24
***************
*** 28,37 ****
  
   o Review already-trained messages, and purge them.
!  o Put in a link to view a message (plain text, html, multipart...?)
!    Include a Reply link that launches the registered email client, eg.
!    mailto:tim@fourstonesExpressions.com?subject=Re:%20pop3proxy&body=Hi%21%0D
!  o [Francois Granger] Show the raw spambrob number close to the buttons
!    (this would mean using the extra X-Hammie header by default).
!  o Add Today and Refresh buttons on the Review page.
  
  User interface improvements:
--- 28,32 ----
  
   o Review already-trained messages, and purge them.
!  o Add a Today button on the Review page.
  
  User interface improvements:
***************
*** 39,49 ****
   o Can it cleanly dynamically update its status display while having a POP3
     conversation?  Hammering reload sucks.
-  o Have both the trained evidence (if present) and current evidence on the
-    show clues page.
  
   o Suggestions?
  """
  
! # This module is part of the spambayes project, which is Copyright 2002
  # The Python Software Foundation and is covered by the Python Software
  # Foundation license.
--- 34,42 ----
   o Can it cleanly dynamically update its status display while having a POP3
     conversation?  Hammering reload sucks.
  
   o Suggestions?
  """
  
! # This module is part of the spambayes project, which is Copyright 2002-3
  # The Python Software Foundation and is covered by the Python Software
  # Foundation license.
***************
*** 62,68 ****
  
  import re
  import time
  import bisect
! import cgi
  
  import tokenizer
--- 55,67 ----
  
  import re
+ import cgi
  import time
+ import types
  import bisect
! 
! try:
!     from sets import Set
! except ImportError:
!     from compatsets import Set
  
  import tokenizer
***************
*** 83,94 ****
      ('pop3proxy',           'remote_servers'),
      ('pop3proxy',           'listen_ports'),
-     ('html_ui',             'display_to'),
-     ('html_ui',             'allow_remote_connections'),
-     ('html_ui',             'http_authentication'),
-     ('html_ui',             'http_user_name'),
-     ('html_ui',             'http_password'),
-     ('Header Options',      None),
-     ('Headers',             'notate_to'),
-     ('Headers',             'notate_subject'),
      ('SMTP Proxy Options',  None),
      ('smtpproxy',           'remote_servers'),
--- 82,85 ----
***************
*** 97,100 ****
--- 88,94 ----
      ('smtpproxy',           'spam_address'),
      ('smtpproxy',           'use_cached_message'),
+     ('Header Options',      None),
+     ('Headers',           'notate_to'),
+     ('Headers',           'notate_subject'),
      ('Storage Options',  None),
      ('Storage',             'persistent_storage_file'),
***************
*** 138,141 ****
--- 132,149 ----
      ('Tokenizer',           'summarize_email_prefixes'),
      ('Tokenizer',           'summarize_email_suffixes'),
+     ('Training Options',   None),
+     ('Hammie',              'train_on_filter'),
+     ('Interface Options',   None),
+     ('html_ui',             'display_headers'),
+     ('html_ui',             'display_received_time'),
+     ('html_ui',             'display_score'),
+     ('html_ui',             'display_adv_find'),
+     ('html_ui',             'default_ham_action'),
+     ('html_ui',             'default_spam_action'),
+     ('html_ui',             'default_unsure_action'),
+     ('html_ui',             'allow_remote_connections'),
+     ('html_ui',             'http_authentication'),
+     ('html_ui',             'http_user_name'),
+     ('html_ui',             'http_password'),
  )
  
***************
*** 150,153 ****
--- 158,162 ----
          self.state_recreator = state_recreator # ugly
          self.app_for_version = "POP3 Proxy"
+         self.previous_sort = None
  
      def onHome(self):
***************
*** 158,161 ****
--- 167,174 ----
          if not state.servers:
              statusTable.proxyDetails = "No POP3 proxies running.
                "
+         findBox = self._buildBox('Word query', 'query.gif',
+                                  self.html.wordQuery)
+         if not options["html_ui", "display_adv_find"]:
+             del findBox.advanced
          content = (self._buildBox('Status and Configuration',
                                    'status.gif', statusTable % stateDict)+
***************
*** 164,169 ****
                     self._buildTrainBox() +
                     self._buildClassifyBox() +
!                    self._buildBox('Word query', 'query.gif',
!                                   self.html.wordQuery) +
                     self._buildBox('Find message', 'query.gif',
                                    self.html.findMessage)
--- 177,181 ----
                     self._buildTrainBox() +
                     self._buildClassifyBox() +
!                    findBox +
                     self._buildBox('Find message', 'query.gif',
                                    self.html.findMessage)
***************
*** 183,187 ****
              messageName = state.getNewMessageName()
              message = state.unknownCorpus.makeMessage(messageName)
!             message.setSubstance(m)
              state.unknownCorpus.addMessage(message)
  
--- 195,199 ----
              messageName = state.getNewMessageName()
              message = state.unknownCorpus.makeMessage(messageName)
!             message.setPayload(m)
              state.unknownCorpus.addMessage(message)
  
***************
*** 213,219 ****
          page or zero if there isn't one, likewise the start of the given page,
          and likewise the start of the next page."""
!         # Fetch all the message keys and sort them into timestamp order.
          allKeys = state.unknownCorpus.keys()
-         allKeys.sort()
  
          # The default start timestamp is derived from the most recent message,
--- 225,230 ----
          page or zero if there isn't one, likewise the start of the given page,
          and likewise the start of the next page."""
!         # Fetch all the message keys
          allKeys = state.unknownCorpus.keys()
  
          # The default start timestamp is derived from the most recent message,
***************
*** 244,271 ****
          return keys, date, prior, start, end
  
!     def _appendMessages(self, table, keyedMessageInfo, label):
          """Appends the rows of a table of messages to 'table'."""
          stripe = 0
!         if not options["html_ui", "display_to"]:
!             del table.to_header
!         nrows = options["html_ui", "rows_per_section"]
!         for key, messageInfo in keyedMessageInfo[:nrows]:
              row = self.html.reviewRow.clone()
              if label == 'Spam':
!                 row.spam.checked = 1
              elif label == 'Ham':
!                 row.ham.checked = 1
              else:
!                 row.defer.checked = 1
!             row.subject = messageInfo.subjectHeader
!             row.subject.title = messageInfo.bodySummary
!             row.subject.href="view?key=%s&corpus=%s" % (key, label)
!             row.from_ = messageInfo.fromHeader
!             if options["html_ui", "display_to"]:
!                 row.to_ = messageInfo.toHeader
              else:
!                 del row.to_
!             subj = cgi.escape(messageInfo.subjectHeader)
              row.classify.href="showclues?key=%s&subject=%s" % (key, subj)
              setattr(row, 'class', ['stripe_on', 'stripe_off'][stripe]) # Grr!
              row = str(row).replace('TYPE', label).replace('KEY', key)
--- 255,331 ----
          return keys, date, prior, start, end
  
!     def _sortMessages(self, messages, sort_order):
!         """Sorts the message by the appropriate attribute.  If this was the
!         previous sort order, then reverse it."""
!         if sort_order is None or sort_order == "received":
!             # Default sorting, which is in reverse order of appearance.
!             # This is complicated because the 'received' info is the key.
!             messages.sort()
!             if self.previous_sort == sort_order:
!                 messages.reverse()
!                 self.previous_sort = None
!             else:
!                 self.previous_sort = 'received'
!             return messages
!         else:
!             tmplist = [(getattr(x[1], sort_order), x) for x in messages]
!         tmplist.sort()
!         if self.previous_sort == sort_order:
!             tmplist.reverse()
!             self.previous_sort = None
!         else:
!             self.previous_sort = sort_order
!         return [x for (key, x) in tmplist]
! 
!     def _appendMessages(self, table, keyedMessageInfo, label, sort_order):
          """Appends the rows of a table of messages to 'table'."""
          stripe = 0
! 
!         keyedMessageInfo = self._sortMessages(keyedMessageInfo, sort_order)
!         for key, messageInfo in keyedMessageInfo:
!             unused, unused, messageInfo.received = \
!                     self._getTimeRange(self._keyToTimestamp(key))
              row = self.html.reviewRow.clone()
              if label == 'Spam':
!                 r_att = getattr(row, options["html_ui",
!                                            "default_spam_action"])
              elif label == 'Ham':
!                 r_att = getattr(row, options["html_ui",
!                                            "default_ham_action"])
              else:
!                 r_att = getattr(row, options["html_ui",
!                                            "default_unsure_action"])
!             setattr(r_att, "checked", 1)
! 
!             row.optionalHeadersValues = '' # make way for real list
!             for header in options["html_ui", "display_headers"]:
!                 header = header.lower()
!                 text = getattr(messageInfo, "%sHeader" % (header,))
!                 if header == "subject":
!                     # Subject is special, because it links to the body.
!                     # If the user doesn't display the subject, then there
!                     # is no link to the body.
!                     h = self.html.reviewRow.linkedHeaderValue.clone()
!                     h.text.title = messageInfo.bodySummary
!                     h.text.href = "view?key=%s&corpus=%s" % (key, label)
!                 else:
!                     h = self.html.reviewRow.headerValue.clone()
!                 h.text = text
!                 row.optionalHeadersValues += h
! 
!             # Apart from any message headers, we may also wish to display
!             # the message score, and the time the message was received.
!             if options["html_ui", "display_score"]:
!                 row.score_ = messageInfo.score
              else:
!                 del row.score_
!             if options["html_ui", "display_received_time"]:
!                 row.received_ = messageInfo.received
!             else:
!                 del row.received_
! 
!             subj = messageInfo.subjectHeader
              row.classify.href="showclues?key=%s&subject=%s" % (key, subj)
+             row.tokens.href="showclues?key=%s&subject=%s&tokens=1" % (key, subj)
              setattr(row, 'class', ['stripe_on', 'stripe_off'][stripe]) # Grr!
              row = str(row).replace('TYPE', label).replace('KEY', key)
***************
*** 350,381 ****
  
          # Else if an id has been specified, just show that message
          elif params.get('find') is not None:
              key = params['find']
              error = False
              if key == "":
                  error = True
!                 page = "
                You must enter an id to find.
                "
!             elif state.unknownCorpus.get(key) == None:
!                 # maybe this message has been moved to the spam
!                 # or ham corpus
!                 if state.hamCorpus.get(key) != None:
!                     sourceCorpus = state.hamCorpus
!                 elif state.spamCorpus.get(key) != None:
!                     sourceCorpus = state.spamCorpus
                  else:
!                     error = True
!                     page = "
                Could not find message with id '"
!                     page += key + "' - maybe it expired.
                "
!             if error == True:
!                 title = "Did not find message"
!                 box = self._buildBox(title, 'status.gif', page)
!                 self.write(box)
!                 self.write(self._buildBox('Find message', 'query.gif',
!                                           self.html.findMessage))
!                 self._writePostamble()
!                 return
!             keys.append(params['find'])
!             prior = this = next = 0
!             title = "Found message"
  
          # Else show the most recent day's page, as decided by _buildReviewKeys.
--- 410,479 ----
  
          # Else if an id has been specified, just show that message
+         # Else if search criteria have been specified, show the messages
+         # that match those criteria.
          elif params.get('find') is not None:
+             prior = this = next = 0
+             keys = Set()        # so we don't end up with duplicates
+             push = keys.add
+             try:
+                 max_results = int(params['max_results'])
+             except ValueError:
+                 max_results = 1
              key = params['find']
+             if params.has_key('ignore_case'):
+                 ic = True
+             else:
+                 ic = False
              error = False
              if key == "":
                  error = True
!                 page = "
                You must enter a search string.
                "
!             else:
!                 if len(keys) < max_results and \
!                    params.has_key('id'):
!                     if state.unknownCorpus.get(key):
!                         push((key, state.unknownCorpus))
!                     elif state.hamCorpus.get(key):
!                         push((key, state.hamCorpus))
!                     elif state.spamCorpus.get(key):
!                         push((key, state.spamCorpus))
!                 if params.has_key('subject') or params.has_key('body') or \
!                    params.has_key('headers'):
!                     # This is an expensive operation, so let the user know
!                     # that something is happening.
!                     self.write('
                Searching...
                ')
!                     for corp in [state.unknownCorpus, state.hamCorpus,
!                                    state.spamCorpus]:
!                         for k in corp.keys():
!                             if len(keys) >= max_results:
!                                 break
!                             msg = corp[k]
!                             msg.load()
!                             if params.has_key('subject'):
!                                 if self._contains(msg['Subject'], key, ic):
!                                     push((k, corp))
!                             if params.has_key('body'):
!                                 msg_body = msg.as_string()
!                                 msg_body = msg_body[msg_body.index('\r\n\r\n'):]
!                                 if self._contains(msg_body, key, ic):
!                                     push((k, corp))
!                             if params.has_key('headers'):
!                                 for nm, val in msg.items():
!                                     if self._contains(nm, key, ic) or \
!                                        self._contains(val, key, ic):
!                                         push((k, corp))
!                 if len(keys):
!                     title = "Found message%s" % (['','s'][len(keys)>1],)
!                     keys = list(keys)
                  else:
!                     page = "
                Could not find any matching messages. " \
!                            "Maybe they expired?
                "
!                     title = "Did not find message"
!                     box = self._buildBox(title, 'status.gif', page)
!                     self.write(box)
!                     self.write(self._buildBox('Find message', 'query.gif',
!                                               self.html.findMessage))
!                     self._writePostamble()
!                     return
  
          # Else show the most recent day's page, as decided by _buildReviewKeys.
***************
*** 391,398 ****
                              }
          for key in keys:
              # Parse the message, get the judgement header and build a message
              # info object for each message.
!             cachedMessage = sourceCorpus[key]
!             message = spambayes.mboxutils.get_message(cachedMessage.getSubstance())
              judgement = message[options["Headers",
                                          "classification_header_name"]]
--- 489,500 ----
                              }
          for key in keys:
+             if isinstance(key, types.TupleType):
+                 key, sourceCorpus = key
+             else:
+                 sourceCorpus = state.unknownCorpus
              # Parse the message, get the judgement header and build a message
              # info object for each message.
!             message = sourceCorpus[key]
!             message.load()
              judgement = message[options["Headers",
                                          "classification_header_name"]]
***************
*** 405,409 ****
  
          # Present the list of messages in their groups in reverse order of
!         # appearance.
          if keys:
              page = self.html.reviewtable.clone()
--- 507,511 ----
  
          # Present the list of messages in their groups in reverse order of
!         # appearance, by default, or according to the specified sort order.
          if keys:
              page = self.html.reviewtable.clone()
***************
*** 415,418 ****
--- 517,521 ----
                  del page.nextButton.disabled
              templateRow = page.reviewRow.clone()
+ 
              page.table = ""  # To make way for the real rows.
              for header, label in ((options["Headers",
***************
*** 424,432 ****
                  messages = keyedMessageInfo[header]
                  if messages:
!                     subHeader = str(self.html.reviewSubHeader)
                      subHeader = subHeader.replace('TYPE', label)
                      page.table += self.html.blankRow
                      page.table += subHeader
!                     self._appendMessages(page.table, messages, label)
  
              page.table += self.html.trainRow
--- 527,549 ----
                  messages = keyedMessageInfo[header]
                  if messages:
!                     sh = self.html.reviewSubHeader.clone()
!                     # Setup the header row
!                     sh.optionalHeaders = ''
!                     h = self.html.headerHeader.clone()
!                     for header in options["html_ui", "display_headers"]:
!                         h.headerLink.href = 'review?sort=%sHeader' % \
!                                             (header.lower(),)
!                         h.headerName = header.title()
!                         sh.optionalHeaders += h
!                     if not options["html_ui", "display_score"]:
!                         del sh.score_header
!                     if not options["html_ui", "display_received_time"]:
!                         del sh.received_header
!                     subHeader = str(sh)
                      subHeader = subHeader.replace('TYPE', label)
                      page.table += self.html.blankRow
                      page.table += subHeader
!                     self._appendMessages(page.table, messages, label,
!                                          params.get('sort'))
  
              page.table += self.html.trainRow
***************
*** 444,447 ****
--- 561,573 ----
          self._writePostamble()
  
+     def _contains(self, a, b, ignore_case=False):
+         """Return true if substring b is part of string a."""
+         assert(isinstance(a, types.StringTypes))
+         assert(isinstance(b, types.StringTypes))
+         if ignore_case:
+             a = a.lower()
+             b = b.lower()
+         return a.find(b) >= 0
+ 
      def onView(self, key, corpus):
          """View a message - linked from the Review page."""
***************
*** 449,464 ****
          message = state.unknownCorpus.get(key)
          if message:
!             self.write("
                %s
                " % cgi.escape(message.getSubstance()))
          else:
              self.write("
                Can't find message %r. Maybe it expired.
                " % key)
          self._writePostamble()
  
!     def onShowclues(self, key, subject):
          """Show clues for a message - linked from the Review page."""
          self._writePreamble("Message clues", parent=('review', 'Review'))
!         message = state.unknownCorpus.get(key).getSubstance()
          message = message.replace('\r\n', '\n').replace('\r', '\n') # For Macs
          if message:
!             results = self._buildCluesTable(message, subject)
              del results.classifyAnother
              self.write(results)
--- 575,591 ----
          message = state.unknownCorpus.get(key)
          if message:
!             self.write("
                %s
                " % cgi.escape(message.as_string()))
          else:
              self.write("
                Can't find message %r. Maybe it expired.
                " % key)
          self._writePostamble()
  
!     def onShowclues(self, key, subject, tokens='0'):
          """Show clues for a message - linked from the Review page."""
+         tokens = bool(int(tokens)) # needs the int, as bool('0') is True
          self._writePreamble("Message clues", parent=('review', 'Review'))
!         message = state.unknownCorpus.get(key).as_string()
          message = message.replace('\r\n', '\n').replace('\r', '\n') # For Macs
          if message:
!             results = self._buildCluesTable(message, subject, tokens)
              del results.classifyAnother
              self.write(results)
***************
*** 469,478 ****
      def _makeMessageInfo(self, message):
          """Given an email.Message, return an object with subjectHeader,
!         fromHeader and bodySummary attributes.  These objects are passed into
!         appendMessages by onReview - passing email.Message objects directly
!         uses too much memory."""
          subjectHeader = message["Subject"] or "(none)"
!         fromHeader = message["From"] or "(none)"
!         toHeader = message["To"] or "(none)"
          try:
              part = typed_subpart_iterator(message, 'text', 'plain').next()
--- 596,625 ----
      def _makeMessageInfo(self, message):
          """Given an email.Message, return an object with subjectHeader,
!         bodySummary and other header (as needed) attributes.  These objects
!         are passed into appendMessages by onReview - passing email.Message
!         objects directly uses too much memory."""
          subjectHeader = message["Subject"] or "(none)"
!         headers = {"subject" : subjectHeader}
!         for header in options["html_ui", "display_headers"]:
!             headers[header.lower()] = (message[header] or "(none)")
!         score = message[options["Headers", "score_header_name"]]
!         if score:
!             # the score might have the log info at the end
!             op = score.find('(')
!             if op >= 0:
!                 score = score[:op]
!             try:
!                 score = "%.2f%%" % (float(score)*100,)
!             except ValueError:
!                 # Hmm.  The score header should only contain a floating
!                 # point number.  What's going on here, then?
!                 score = "Err"  # Let the user know something is wrong.
!         else:
!             # If the lookup fails, this means that the "include_score"
!             # option isn't activated. We have the choice here to either
!             # calculate it now, which is pretty inefficient, since we have
!             # already done so, or to admit that we don't know what it is.
!             # We'll go with the latter.
!             score = "?"
          try:
              part = typed_subpart_iterator(message, 'text', 'plain').next()
***************
*** 500,506 ****
              pass
          messageInfo = _MessageInfo()
!         messageInfo.subjectHeader = self._trimHeader(subjectHeader, 50, True)
!         messageInfo.fromHeader = self._trimHeader(fromHeader, 40, True)
!         messageInfo.toHeader = self._trimHeader(toHeader, 40, True)
          messageInfo.bodySummary = self._trimHeader(text, 200)
          return messageInfo
--- 647,654 ----
              pass
          messageInfo = _MessageInfo()
!         for headerName, headerValue in headers.items():
!             headerValue = self._trimHeader(headerValue, 45, True)
!             setattr(messageInfo, "%sHeader" % (headerName,), headerValue)
!         messageInfo.score = score
          messageInfo.bodySummary = self._trimHeader(text, 200)
          return messageInfo

Index: UserInterface.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/UserInterface.py,v
retrieving revision 1.24
retrieving revision 1.25
diff -C2 -d -r1.24 -r1.25
*** UserInterface.py	8 Sep 2003 07:04:17 -0000	1.24
--- UserInterface.py	19 Sep 2003 23:38:10 -0000	1.25
***************
*** 262,355 ****
          self._writePostamble()
  
!     def _buildCluesTable(self, message, subject=None):
          cluesTable = self.html.cluesTable.clone()
          cluesRow = cluesTable.cluesRow.clone()
          del cluesTable.cluesRow   # Delete dummy row to make way for real ones
!         (probability, clues) = classifier.spamprob(tokenizer.tokenize(message),\
!                                                     evidence=True)
          for word, wordProb in clues:
!             cluesTable += cluesRow % (cgi.escape(word), wordProb)
  
          results = self.html.classifyResults.clone()
!         results.probability = probability
          if subject is None:
!             heading = "Clues:"
          else:
!             heading = "Clues for: " + subject
          results.cluesBox = self._buildBox(heading, 'status.gif', cluesTable)
          return results
  
!     def onWordquery(self, word):
!         wildcard_limit = 10
!         statsBoxes = []
          if word == "":
!             stats = "You must enter a word."
!             statsBoxes.append(self._buildBox("Statistics for %r" % \
!                                              cgi.escape(word),
!                                              'status.gif', stats))
          else:
!             word = word.lower()
!             if word[-1] == '*':
!                 # Wildcard search - list all words that start with word[:-1]
!                 word = word[:-1]
!                 reached_limit = False
!                 for w in classifier._wordinfokeys():
!                     if not reached_limit and len(statsBoxes) > wildcard_limit:
!                         reached_limit = True
!                         over_limit = 0
!                     if w.startswith(word):
!                         if reached_limit:
!                             over_limit += 1
!                         else:
!                             wordinfo = classifier._wordinfoget(w)
!                             stats = self.html.wordStats.clone()
!                             stats.spamcount = wordinfo.spamcount
!                             stats.hamcount = wordinfo.hamcount
!                             stats.spamprob = classifier.probability(wordinfo)
!                             box = self._buildBox("Statistics for %r" % \
!                                                  cgi.escape(w),
!                                                  'status.gif', stats)
!                             statsBoxes.append(box)
!                 if len(statsBoxes) == 0:
!                     stats = "There are no words that begin with '%s' " \
!                             "in the database." % (word,)
!                     # We build a box for each word; I'm not sure this is
!                     # produces the nicest results, but it's ok with a
!                     # limited number of words.
!                     statsBoxes.append(self._buildBox("Statistics for %s" % \
!                                                      cgi.escape(word),
!                                                      'status.gif', stats))
!                 elif reached_limit:
!                     if over_limit == 1:
!                         singles = ["was", "match", "is"]
                      else:
!                         singles = ["were", "matches", "are"]
!                     stats = "There %s %d additional %s that %s not " \
!                             "shown here." % (singles[0], over_limit,
!                                              singles[1], singles[2])
!                     box = self._buildBox("Statistics for '%s*'" % \
!                                          cgi.escape(word), 'status.gif',
!                                          stats)
!                     statsBoxes.append(box)
!             else:
!                 # Optimised version for non-wildcard searches
!                 wordinfo = classifier._wordinfoget(word)
!                 if wordinfo:
!                     stats = self.html.wordStats.clone()
!                     stats.spamcount = wordinfo.spamcount
!                     stats.hamcount = wordinfo.hamcount
!                     stats.spamprob = classifier.probability(wordinfo)
                  else:
!                     stats = "%r does not exist in the database." % cgi.escape(word)
!                 statsBoxes.append(self._buildBox("Statistics for %r" % \
!                                                  cgi.escape(word),
!                                                  'status.gif', stats))
  
-         query = self.html.wordQuery.clone()
-         query.word.value = "%s" % (word,)
-         queryBox = self._buildBox("Word query", 'query.gif', query)
          self._writePreamble("Word query")
!         for box in statsBoxes:
!             self.write(box)
          self.write(queryBox)
          self._writePostamble()
--- 262,464 ----
          self._writePostamble()
  
!     ev_re = re.compile("%s:(.*?)(?:\n\S|\n\n)" % \
!                        re.escape(options["Headers",
!                                          "evidence_header_name"]),
!                        re.DOTALL)
!     sc_re = re.compile("%s:(.*)\n" % \
!                        re.escape(options["Headers", "score_header_name"]))
! 
!     def _fillCluesTable(self, clues):
!         accuracy = 6
          cluesTable = self.html.cluesTable.clone()
          cluesRow = cluesTable.cluesRow.clone()
          del cluesTable.cluesRow   # Delete dummy row to make way for real ones
!         fetchword = classifier._wordinfoget
          for word, wordProb in clues:
!             record = fetchword(word)
!             if record:
!                 nham = record.hamcount
!                 nspam = record.spamcount
!                 if wordProb is None:
!                     wordProb = classifier.probability(record)
!             elif word != "*H*" and word != "*S*":
!                 nham = nspam = 0
!             else:
!                 nham = nspam = "-"
!             if wordProb is None:
!                 wordProb = "-"
!             else:
!                 wordProb = round(float(wordProb), accuracy)
!             cluesTable += cluesRow % (cgi.escape(word), wordProb,
!                                       nham, nspam)
!         return cluesTable
!     
!     def _buildCluesTable(self, message, subject=None, show_tokens=False):
!         tokens = tokenizer.tokenize(message)
!         if show_tokens:
!             clues = []
!             for tok in tokens:
!                 clues.append((tok, None))
!             probability = classifier.spamprob(tokens)
!             cluesTable = self._fillCluesTable(clues)
!             head_name = "Tokens"
!         else:
!             (probability, clues) = classifier.spamprob(tokens, evidence=True)
!             cluesTable = self._fillCluesTable(clues)
!             head_name = "Clues"
  
          results = self.html.classifyResults.clone()
!         results.probability = "%.2f%% (%s)" % (probability*100, probability)
          if subject is None:
!             heading = "%s: (%s)" % (head_name, len(clues))
          else:
!             heading = "%s for: %s (%s)" % (head_name, subject, len(clues))
          results.cluesBox = self._buildBox(heading, 'status.gif', cluesTable)
+         if not show_tokens:
+             mo = self.sc_re.search(message)
+             if mo:
+                 # Also display the score the message received when it was
+                 # classified.
+                 prob = float(mo.group(1).strip())
+                 results.orig_prob_num = "%.2f%% (%s)" % (prob*100, prob)
+             else:
+                 del results.orig_prob
+             mo = self.ev_re.search(message)
+             if mo:
+                 # Also display the clues as they were when the message was
+                 # classified.
+                 clues = []
+                 evidence = mo.group(1).strip().split(';')
+                 for clue in evidence:
+                     word, prob = clue.strip().split(': ')
+                     clues.append((word.strip("'"), prob))
+                 cluesTable = self._fillCluesTable(clues)
+ 
+                 if subject is None:
+                     heading = "Original clues: (%s)" % (len(evidence),)
+                 else:
+                     heading = "Original clues for: %s (%s)" % (subject,
+                                                                len(evidence),)
+                 orig_results = self._buildBox(heading, 'status.gif',
+                                               cluesTable)
+                 results.cluesBox += orig_results
+         else:
+             del results.orig_prob
          return results
  
!     def onWordquery(self, word, query_type="basic", max_results='10',
!                     ignore_case=False):
!         # It would be nice if the default value for max_results here
!         # always matched the value in ui.html.
!         try:
!             max_results = int(max_results)
!         except ValueError:
!             # Ignore any invalid number, like "foo"
!             max_results = 10
! 
!         original_word = word
! 
!         query = self.html.wordQuery.clone()
!         query.word.value = "%s" % (word,)
!         for q_type in [query.advanced.basic,
!                                query.advanced.wildcard,
!                                query.advanced.regex]:
!             if query_type == q_type.id:
!                 q_type.checked = 'checked'
!                 if query_type != "basic":
!                     del query.advanced.max_results.disabled
!         if ignore_case:
!             query.advanced.ignore_case.checked = 'checked'
!         query.advanced.max_results.value = str(max_results)
!         queryBox = self._buildBox("Word query", 'query.gif', query)
!         if not options["html_ui", "display_adv_find"]:
!             del queryBox.advanced
! 
!         stats = []
          if word == "":
!             stats.append("You must enter a word.")
!         elif query_type == "basic" and not ignore_case:
!             wordinfo = classifier._wordinfoget(word)
!             if wordinfo:
!                 stat = (word, wordinfo.spamcount, wordinfo.hamcount,
!                         classifier.probability(wordinfo))
!             else:
!                 stat = "%r does not exist in the database." % \
!                        cgi.escape(word)
!             stats.append(stat)
          else:
!             if query_type != "regex":
!                 word = re.escape(word)
!             if query_type == "wildcard":
!                 word = word.replace("\\?", ".")
!                 word = word.replace("\\*", ".*")
! 
!             flags = 0
!             if ignore_case:
!                 flags = re.IGNORECASE
!             r = re.compile(word, flags)
! 
!             reached_limit = False
!             for w in classifier._wordinfokeys():
!                 if not reached_limit and len(stats) >= max_results:
!                     reached_limit = True
!                     over_limit = 0
!                 if r.match(w):
!                     if reached_limit:
!                         over_limit += 1
                      else:
!                         wordinfo = classifier._wordinfoget(w)
!                         stat = (w, wordinfo.spamcount, wordinfo.hamcount,
!                                 classifier.probability(wordinfo))
!                         stats.append(stat)
!             if len(stats) == 0 and max_results > 0:
!                 stat = "There are no words that begin with '%s' " \
!                         "in the database." % (word,)
!                 stats.append(stat)
!             elif reached_limit:
!                 if over_limit == 1:
!                     singles = ["was", "match", "is"]
                  else:
!                     singles = ["were", "matches", "are"]
!                 stat = "There %s %d additional %s that %s not " \
!                        "shown here." % (singles[0], over_limit,
!                                         singles[1], singles[2])
!                 stats.append(stat)
  
          self._writePreamble("Word query")
!         if len(stats) == 1:
!             if isinstance(stat, types.TupleType):
!                 stat = self.html.wordStats.clone()
!                 word = stats[0][0]
!                 stat.spamcount = stats[0][1]
!                 stat.hamcount = stats[0][2]
!                 stat.spamprob = "%.6f" % stats[0][3]
!             else:
!                 stat = stats[0]
!                 word = original_word
!             row = self._buildBox("Statistics for '%s'" % \
!                                  cgi.escape(word),
!                                  'status.gif', stat)
!             self.write(row)
!         else:
!             page = self.html.multiStats.clone()
!             page.multiTable = "" # make way for the real rows
!             page.multiTable += self.html.multiHeader.clone()
!             stripe = 0
!             for stat in stats:
!                 if isinstance(stat, types.TupleType):
!                     row = self.html.statsRow.clone()
!                     row.word, row.spamcount, row.hamcount = stat[:3]
!                     row.spamprob = "%.6f" % stat[3]
!                     setattr(row, 'class', ['stripe_on', 'stripe_off'][stripe])
!                     stripe = stripe ^ 1
!                     page.multiTable += row
!                 else:
!                     self.write(self._buildBox("Statistics for '%s'" % \
!                                               cgi.escape(original_word),
!                                               'status.gif', stat))
!             self.write(self._buildBox("Statistics for '%s'" % \
!                                       cgi.escape(original_word), 'status.gif',
!                                       page))
          self.write(queryBox)
          self._writePostamble()

Index: mboxutils.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/mboxutils.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** mboxutils.py	14 Jan 2003 05:38:20 -0000	1.2
--- mboxutils.py	19 Sep 2003 23:38:10 -0000	1.3
***************
*** 106,109 ****
--- 106,115 ----
      (everything through the first blank line) are thrown out, and the
      rest of the text is wrapped in a bare email.Message.Message.
+ 
+     Note that we can't use our own message class here, because this
+     function is imported by tokenizer, and our message class imports
+     tokenizer, so we get a circular import problem.  In any case, this
+     function does need anything that our message class offers, so that
+     shouldn't matter.
      """
  

Index: message.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/message.py,v
retrieving revision 1.37
retrieving revision 1.38
diff -C2 -d -r1.37 -r1.38
*** message.py	18 Sep 2003 03:58:59 -0000	1.37
--- message.py	19 Sep 2003 23:38:10 -0000	1.38
***************
*** 88,107 ****
          return not not val
  
- import sys
  import os
  import types
  import re
  
! import email            # for message_from_string
  import email.Message
  import email.Parser
  
! from spambayes.tokenizer import tokenize
  from spambayes.Options import options
  
  from cStringIO import StringIO
  
- from spambayes import dbmstorage
- import shelve
  
  CRLF_RE = re.compile(r'\r\n|\r|\n')
--- 88,109 ----
          return not not val
  
  import os
  import types
+ import math
  import re
+ import sys
+ import types
+ import shelve
  
! import email
  import email.Message
  import email.Parser
  
! from spambayes import dbmstorage
  from spambayes.Options import options
+ from spambayes.tokenizer import tokenize
  
  from cStringIO import StringIO
  
  
  CRLF_RE = re.compile(r'\r\n|\r|\n')
***************
*** 286,290 ****
  
          if options['Headers','include_score']:
!             self[options['Headers','score_header_name']] = str(prob)
  
          if options['Headers','include_thermostat']:
--- 288,300 ----
  
          if options['Headers','include_score']:
!             disp = str(prob)
!             if options["Headers", "header_score_logarithm"]:
!                 if prob<=0.005 and prob>0.0:
!                     x=-math.log10(prob)
!                     disp += " (%d)"%x
!                 if prob>=0.995 and prob<1.0:
!                     x=-math.log10(1.0-prob)
!                     disp += " (%d)"%x
!             self[options['Headers','score_header_name']] = disp
  
          if options['Headers','include_thermostat']:



From anadelonbrin at users.sourceforge.net  Sat Sep 20 01:01:10 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sat Sep 20 01:01:15 2003
Subject: [Spambayes-checkins] spambayes README-DEVEL.txt,1.4,1.5
Message-ID: 

Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv30634

Modified Files:
	README-DEVEL.txt 
Log Message:
Add a note about updating the autoresponder.

Index: README-DEVEL.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/README-DEVEL.txt,v
retrieving revision 1.4
retrieving revision 1.5
diff -C2 -d -r1.4 -r1.5
*** README-DEVEL.txt	5 Sep 2003 08:04:51 -0000	1.4
--- README-DEVEL.txt	20 Sep 2003 05:01:07 -0000	1.5
***************
*** 358,361 ****
--- 358,364 ----
     existing tag names for the tag name format.
   o Update the website News, Download and Application sections.
+  o Update reply.txt in the website repository as needed (it specifies the
+    latest version).  Then let Tim, Barry or Skip know that they need to
+    update the autoresponder.
  
  Then announce the release on the mailing lists and watch the bug reports



From anadelonbrin at users.sourceforge.net  Sat Sep 20 01:03:09 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sat Sep 20 01:03:12 2003
Subject: [Spambayes-checkins] website reply.txt,1.7,1.8
Message-ID: 

Update of /cvsroot/spambayes/website
In directory sc8-pr-cvs1:/tmp/cvs-serv30902

Modified Files:
	reply.txt 
Log Message:
Update to the latest versions.

Index: reply.txt
===================================================================
RCS file: /cvsroot/spambayes/website/reply.txt,v
retrieving revision 1.7
retrieving revision 1.8
diff -C2 -d -r1.7 -r1.8
*** reply.txt	28 Aug 2003 00:05:41 -0000	1.7
--- reply.txt	20 Sep 2003 05:03:07 -0000	1.8
***************
*** 39,43 ****
  
  Please ensure that you have the latest version.  As of 2003-08-28, this is
! 007 for the Outlook plug-in, and 1.0a4 for everything else.  If you are
  still having trouble, try looking at the bug reports that are currently
  open:
--- 39,43 ----
  
  Please ensure that you have the latest version.  As of 2003-08-28, this is
! 008.1 for the Outlook plug-in, and 1.0a6 for everything else.  If you are
  still having trouble, try looking at the bug reports that are currently
  open:



From tim_one at users.sourceforge.net  Sat Sep 20 19:23:14 2003
From: tim_one at users.sourceforge.net (Tim Peters)
Date: Sat Sep 20 19:23:18 2003
Subject: [Spambayes-checkins] website reply.txt,1.8,1.9
Message-ID: 

Update of /cvsroot/spambayes/website
In directory sc8-pr-cvs1:/tmp/cvs-serv8204

Modified Files:
	reply.txt 
Log Message:
Update the date.


Index: reply.txt
===================================================================
RCS file: /cvsroot/spambayes/website/reply.txt,v
retrieving revision 1.8
retrieving revision 1.9
diff -C2 -d -r1.8 -r1.9
*** reply.txt	20 Sep 2003 05:03:07 -0000	1.8
--- reply.txt	20 Sep 2003 23:23:12 -0000	1.9
***************
*** 38,42 ****
  -----------------------------------------------
  
! Please ensure that you have the latest version.  As of 2003-08-28, this is
  008.1 for the Outlook plug-in, and 1.0a6 for everything else.  If you are
  still having trouble, try looking at the bug reports that are currently
--- 38,42 ----
  -----------------------------------------------
  
! Please ensure that you have the latest version.  As of 2003-09-20, this is
  008.1 for the Outlook plug-in, and 1.0a6 for everything else.  If you are
  still having trouble, try looking at the bug reports that are currently



From anadelonbrin at users.sourceforge.net  Sun Sep 21 00:21:59 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sun Sep 21 00:22:02 2003
Subject: [Spambayes-checkins] spambayes MANIFEST.in,1.7,1.7.2.1
Message-ID: 

Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv16417

Modified Files:
      Tag: release_1_0
	MANIFEST.in 
Log Message:
Argh - how did I miss this?  We didn't list the scripts directory in the manifest.
So why did the zip include it?

Index: MANIFEST.in
===================================================================
RCS file: /cvsroot/spambayes/spambayes/MANIFEST.in,v
retrieving revision 1.7
retrieving revision 1.7.2.1
diff -C2 -d -r1.7 -r1.7.2.1
*** MANIFEST.in	18 Sep 2003 22:05:59 -0000	1.7
--- MANIFEST.in	21 Sep 2003 04:21:57 -0000	1.7.2.1
***************
*** 5,8 ****
--- 5,9 ----
  recursive-include Outlook2000 *.py *.txt *.ini *.html *.bmp *.rc *.h
  recursive-include utilities *.py *.txt
+ recursive-include scripts *.py *.txt
  recursive-include testtools *.py *.txt
  recursive-include windows *.py *.txt *.h *.rc *.ico



From anadelonbrin at users.sourceforge.net  Sun Sep 21 00:22:21 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sun Sep 21 00:22:23 2003
Subject: [Spambayes-checkins] spambayes MANIFEST.in,1.7,1.8
Message-ID: 

Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv16478

Modified Files:
	MANIFEST.in 
Log Message:
Argh - how did I miss this?  We didn't list the scripts directory in the manifest.
So why did the zip include it?

Index: MANIFEST.in
===================================================================
RCS file: /cvsroot/spambayes/spambayes/MANIFEST.in,v
retrieving revision 1.7
retrieving revision 1.8
diff -C2 -d -r1.7 -r1.8
*** MANIFEST.in	18 Sep 2003 22:05:59 -0000	1.7
--- MANIFEST.in	21 Sep 2003 04:22:19 -0000	1.8
***************
*** 5,8 ****
--- 5,9 ----
  recursive-include Outlook2000 *.py *.txt *.ini *.html *.bmp *.rc *.h
  recursive-include utilities *.py *.txt
+ recursive-include scripts *.py *.txt
  recursive-include testtools *.py *.txt
  recursive-include windows *.py *.txt *.h *.rc *.ico



From anadelonbrin at users.sourceforge.net  Sun Sep 21 00:24:47 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sun Sep 21 00:24:49 2003
Subject: [Spambayes-checkins] spambayes setup.py,1.24,1.25
Message-ID: 

Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv16699

Modified Files:
	setup.py 
Log Message:
sb_smtpproxy.py is no longer a script, so don't try and install it.  Do offer to remove it, though.

Index: setup.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/setup.py,v
retrieving revision 1.24
retrieving revision 1.25
diff -C2 -d -r1.24 -r1.25
*** setup.py	19 Sep 2003 13:14:20 -0000	1.24
--- setup.py	21 Sep 2003 04:24:44 -0000	1.25
***************
*** 38,41 ****
--- 38,42 ----
          'pop3proxy',
          'smtpproxy',
+         'sb_smtpproxy',
          'proxytee',
          'dbExpImp',
***************
*** 85,89 ****
               'scripts/sb_pop3dnd.py',
               'scripts/sb_server.py',
-              'scripts/sb_smtpproxy.py',
               'scripts/sb_unheader.py',
               'scripts/sb_upload.py',
--- 86,89 ----



From anadelonbrin at users.sourceforge.net  Sun Sep 21 00:29:41 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sun Sep 21 00:29:43 2003
Subject: [Spambayes-checkins] spambayes setup.py,1.24,1.24.2.1
Message-ID: 

Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv17290

Modified Files:
      Tag: release_1_0
	setup.py 
Log Message:
2.2 compatibility for the install script.

Index: setup.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/setup.py,v
retrieving revision 1.24
retrieving revision 1.24.2.1
diff -C2 -d -r1.24 -r1.24.2.1
*** setup.py	19 Sep 2003 13:14:20 -0000	1.24
--- setup.py	21 Sep 2003 04:29:38 -0000	1.24.2.1
***************
*** 24,27 ****
--- 24,32 ----
      DistributionMetadata.classifiers = None
  
+ try:
+     True, False
+ except NameError:
+     # Maintain compatibility with Python 2.2
+     True, False = 1, 0
  
  from spambayes import __version__



From anadelonbrin at users.sourceforge.net  Sun Sep 21 00:30:10 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sun Sep 21 00:30:13 2003
Subject: [Spambayes-checkins] spambayes setup.py,1.25,1.26
Message-ID: 

Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv17391

Modified Files:
	setup.py 
Log Message:
2.2 compatibility for the install script.

Index: setup.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/setup.py,v
retrieving revision 1.25
retrieving revision 1.26
diff -C2 -d -r1.25 -r1.26
*** setup.py	21 Sep 2003 04:24:44 -0000	1.25
--- setup.py	21 Sep 2003 04:30:08 -0000	1.26
***************
*** 24,27 ****
--- 24,32 ----
      DistributionMetadata.classifiers = None
  
+ try:
+     True, False
+ except NameError:
+     # Maintain compatibility with Python 2.2
+     True, False = 1, 0
  
  from spambayes import __version__



From anadelonbrin at users.sourceforge.net  Sun Sep 21 03:01:10 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sun Sep 21 03:01:16 2003
Subject: [Spambayes-checkins] spambayes/spambayes Options.py,1.80,1.81
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv3483/spambayes

Modified Files:
	Options.py 
Log Message:
2.2 compatibility.

Index: Options.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v
retrieving revision 1.80
retrieving revision 1.81
diff -C2 -d -r1.80 -r1.81
*** Options.py	19 Sep 2003 23:38:10 -0000	1.80
--- Options.py	21 Sep 2003 07:01:07 -0000	1.81
***************
*** 20,23 ****
--- 20,29 ----
  import sys, os
  
+ try:
+     True, False
+ except NameError:
+     # Maintain compatibility with Python 2.2
+     True, False = 1, 0
+ 
  __all__ = ['options']
  
***************
*** 830,834 ****
       """Number of rows to display per ham/spam/unsure section.""",
       INTEGER, RESTORE),
- 
    ),
  
--- 836,839 ----



From anadelonbrin at users.sourceforge.net  Sun Sep 21 03:01:34 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sun Sep 21 03:01:38 2003
Subject: [Spambayes-checkins] spambayes/spambayes Options.py,1.79,1.79.2.1
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv3538/spambayes

Modified Files:
      Tag: release_1_0
	Options.py 
Log Message:
2.2 compatibility.

Index: Options.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v
retrieving revision 1.79
retrieving revision 1.79.2.1
diff -C2 -d -r1.79 -r1.79.2.1
*** Options.py	18 Sep 2003 13:55:11 -0000	1.79
--- Options.py	21 Sep 2003 07:01:32 -0000	1.79.2.1
***************
*** 20,23 ****
--- 20,29 ----
  import sys, os
  
+ try:
+     True, False
+ except NameError:
+     # Maintain compatibility with Python 2.2
+     True, False = 1, 0
+ 
  __all__ = ['options']
  



From mhammond at users.sourceforge.net  Mon Sep 22 01:21:15 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Mon Sep 22 01:21:19 2003
Subject: [Spambayes-checkins] spambayes/windows/py2exe setup_all.py,1.7,1.8
Message-ID: 

Update of /cvsroot/spambayes/spambayes/windows/py2exe
In directory sc8-pr-cvs1:/tmp/cvs-serv30809

Modified Files:
	setup_all.py 
Log Message:
We changed the way py2exe handles options.


Index: setup_all.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/py2exe/setup_all.py,v
retrieving revision 1.7
retrieving revision 1.8
diff -C2 -d -r1.7 -r1.8
*** setup_all.py	19 Sep 2003 05:42:04 -0000	1.7
--- setup_all.py	22 Sep 2003 05:21:13 -0000	1.8
***************
*** 29,42 ****
  import py2exe
  
! class Options:
!     def __init__(self, **kw):
!         self.__dict__.update(kw)
! 
! # py2exe_options is a global name found by py2exe
! py2exe_options = Options(
      packages = "spambayes.resources,encodings",
      excludes = "win32ui,pywin,pywin.debugger", # pywin is a package, and still seems to be included.
      includes = "dialogs.resources.dialogs", # Outlook dynamic dialogs
!     dll_excludes = ["dapi.dll", "mapi32.dll"],
      lib_dir = "lib",
      typelibs = [
--- 29,37 ----
  import py2exe
  
! py2exe_options = dict(
      packages = "spambayes.resources,encodings",
      excludes = "win32ui,pywin,pywin.debugger", # pywin is a package, and still seems to be included.
      includes = "dialogs.resources.dialogs", # Outlook dynamic dialogs
!     dll_excludes = "dapi.dll,mapi32.dll",
      lib_dir = "lib",
      typelibs = [
***************
*** 59,63 ****
  
  # These are just objects passed to py2exe
! outlook_addin = Options(
      modules = ["addin"],
      dest_base = "outlook/spambayes_addin",
--- 54,58 ----
  
  # These are just objects passed to py2exe
! outlook_addin = dict(
      modules = ["addin"],
      dest_base = "outlook/spambayes_addin",
***************
*** 69,86 ****
  #    bitmap_resources = outlook_bmp_resources,
  #)
! outlook_dump_props = Options(
      script = os.path.join(sb_top_dir, r"Outlook2000\sandbox\dump_props.py"),
      dest_base = "outlook/outlook_dump_props",
  )
  
! service = Options(
      dest_base = "proxy/pop3proxy_service",
      modules = ["pop3proxy_service"]
  )
! sb_server = Options(
      dest_base = "proxy/sb_server",
      script = os.path.join(sb_top_dir, "scripts", "sb_server.py")
  )
! pop3proxy_tray = Options(
      dest_base = "proxy/pop3proxy_tray",
      script = os.path.join(sb_top_dir, "windows", "pop3proxy_tray.py"),
--- 64,85 ----
  #    bitmap_resources = outlook_bmp_resources,
  #)
! outlook_dump_props = dict(
      script = os.path.join(sb_top_dir, r"Outlook2000\sandbox\dump_props.py"),
      dest_base = "outlook/outlook_dump_props",
  )
  
! service = dict(
      dest_base = "proxy/pop3proxy_service",
      modules = ["pop3proxy_service"]
  )
! sb_server = dict(
      dest_base = "proxy/sb_server",
      script = os.path.join(sb_top_dir, "scripts", "sb_server.py")
  )
! sb_upload = dict(
!     dest_base = "proxy/sb_upload",
!     script = os.path.join(sb_top_dir, "scripts", "sb_upload.py")
! )
! pop3proxy_tray = dict(
      dest_base = "proxy/pop3proxy_tray",
      script = os.path.join(sb_top_dir, "windows", "pop3proxy_tray.py"),
***************
*** 111,118 ****
        service=[service],
        # console exes for debugging
!       console=[sb_server, outlook_dump_props],
        # The taskbar
        windows=[pop3proxy_tray],
        # and the misc data files
        data_files = outlook_data_files + proxy_data_files,
  )
--- 110,118 ----
        service=[service],
        # console exes for debugging
!       console=[sb_server, sb_upload, outlook_dump_props],
        # The taskbar
        windows=[pop3proxy_tray],
        # and the misc data files
        data_files = outlook_data_files + proxy_data_files,
+       options = {"py2exe" : py2exe_options},
  )



From mhammond at users.sourceforge.net  Mon Sep 22 04:59:00 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Mon Sep 22 04:59:03 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py,1.12,1.13
Message-ID: 

Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv1329

Modified Files:
	pop3proxy_tray.py 
Log Message:
Delete old comments no longer relevant.


Index: pop3proxy_tray.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_tray.py,v
retrieving revision 1.12
retrieving revision 1.13
diff -C2 -d -r1.12 -r1.13
*** pop3proxy_tray.py	19 Sep 2003 22:48:29 -0000	1.12
--- pop3proxy_tray.py	22 Sep 2003 08:58:57 -0000	1.13
***************
*** 376,383 ****
  
      def StartStop(self):
-         # XXX This needs to be finished off.
-         # XXX This should determine if we are using the service, and if so
-         # XXX start/stop that, and if not kick sb_server off in a separate
-         # XXX thread, or stop the thread that was started.
          if self.use_service:
              if self.GetServiceStatus() in stoppedStatus:
--- 376,379 ----
***************
*** 398,403 ****
              self.control_functions[START_STOP_ID] = ("Start SpamBayes",
                                           self.StartStop)
- 
- 
  
      def OpenInterface(self):
--- 394,397 ----



From xenogeist at users.sourceforge.net  Tue Sep 23 21:30:57 2003
From: xenogeist at users.sourceforge.net (Adam Walker)
Date: Tue Sep 23 21:31:02 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py,1.13,1.14
Message-ID: 

Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv24999/windows

Modified Files:
	pop3proxy_tray.py 
Log Message:
Check if the web interface port can be bound as a simple test of if the proxy is running.

Index: pop3proxy_tray.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_tray.py,v
retrieving revision 1.13
retrieving revision 1.14
diff -C2 -d -r1.13 -r1.14
*** pop3proxy_tray.py	22 Sep 2003 08:58:57 -0000	1.13
--- pop3proxy_tray.py	24 Sep 2003 01:30:55 -0000	1.14
***************
*** 26,29 ****
--- 26,30 ----
  import thread
  import traceback
+ import socket
  
  # This should just be imported from dialogs.dlgutils, but
***************
*** 170,174 ****
              "SpamBayes")
          Shell_NotifyIcon(NIM_ADD, nid)
!         self.started = False
          self.tip = None
        
--- 171,175 ----
              "SpamBayes")
          Shell_NotifyIcon(NIM_ADD, nid)
!         self.started = self.IsPortBound()
          self.tip = None
        
***************
*** 195,198 ****
--- 196,212 ----
              self.StartStop()
  
+     def IsPortBound(self):        
+         s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+         s.settimeout(0.25)
+         inuse = False
+         try:
+             s.bind(("0.0.0.0",options["html_ui", "port"],))
+         except:
+             inuse = True
+         s.close()
+         s = None
+         return inuse
+ 
+ 
      def BuildToolTip(self):
          tip = None
***************
*** 211,215 ****
  
      def UpdateIcon(self):
!         flags = NIF_TIP | NIF_ICON
          if self.started:
              hicon = self.hstartedicon 
--- 225,229 ----
  
      def UpdateIcon(self):
!         flags = NIF_TIP | NIF_ICON        
          if self.started:
              hicon = self.hstartedicon 
***************
*** 218,221 ****
--- 232,241 ----
          self.tip = self.BuildToolTip()
          nid = (self.hwnd, 0, flags, WM_TASKBAR_NOTIFY, hicon, self.tip)
+         if self.started:
+             self.control_functions[START_STOP_ID] = ("Stop SpamBayes",
+                                                      self.StartStop)
+         else:
+             self.control_functions[START_STOP_ID] = ("Start SpamBayes",
+                                          self.StartStop)
          Shell_NotifyIcon(NIM_MODIFY, nid)
  
***************
*** 320,324 ****
      def OnTaskbarNotify(self, hwnd, msg, wparam, lparam):
          if lparam==win32con.WM_MOUSEMOVE:
!             if self.tip != self.BuildToolTip():
                  self.UpdateIcon()
          if lparam==win32con.WM_LBUTTONUP:
--- 340,345 ----
      def OnTaskbarNotify(self, hwnd, msg, wparam, lparam):
          if lparam==win32con.WM_MOUSEMOVE:
!             if self.tip != self.BuildToolTip() or self.started != self.IsPortBound():
!                 self.started = self.IsPortBound()
                  self.UpdateIcon()
          if lparam==win32con.WM_LBUTTONUP:
***************
*** 388,397 ****
                  self.StartProxyThread()
          self.UpdateIcon()
-         if self.started:
-             self.control_functions[START_STOP_ID] = ("Stop SpamBayes",
-                                                      self.StartStop)
-         else:
-             self.control_functions[START_STOP_ID] = ("Start SpamBayes",
-                                          self.StartStop)
  
      def OpenInterface(self):
--- 409,412 ----



From anadelonbrin at users.sourceforge.net  Tue Sep 23 23:54:17 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Tue Sep 23 23:54:23 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_server.py,1.6,1.6.2.1
Message-ID: 

Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv16488/scripts

Modified Files:
      Tag: release_1_0
	sb_server.py 
Log Message:
Stupid global variables!

Thanks to a global variable not being updated, when we recreated everything, the userinterface kept using the old classifier.
Since we now behave and close that one, this caused all sorts of problems.  Get rid of the damn global variable, and correctly
update it, and all is well in the world again.

In addition, don't save an empty database.  I think we make assumptions about the db being non-empty in some places.

This should fix [ spambayes-Bugs-809769 ] TypeError when training 1.0a6

(I can't believe it took so long for me to find this!)

Index: sb_server.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_server.py,v
retrieving revision 1.6
retrieving revision 1.6.2.1
diff -C2 -d -r1.6 -r1.6.2.1
*** sb_server.py	18 Sep 2003 03:58:59 -0000	1.6
--- sb_server.py	24 Sep 2003 03:54:14 -0000	1.6.2.1
***************
*** 705,708 ****
--- 705,709 ----
  
  state = State()
+ 
  proxyListeners = []
  def _createProxies(servers, proxyPorts):
***************
*** 725,729 ****
      # complains if we try to reopen it without closing it first.
      if hasattr(state, "bayes"):
!         state.bayes.store()
          state.bayes.close()
  
--- 726,732 ----
      # complains if we try to reopen it without closing it first.
      if hasattr(state, "bayes"):
!         # Only store a non-empty db.
!         if state.bayes.nham != 0 and state.bayes.nspam != 0:
!             state.bayes.store()
          state.bayes.close()
  



From anadelonbrin at users.sourceforge.net  Tue Sep 23 23:54:17 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Tue Sep 23 23:54:28 2003
Subject: [Spambayes-checkins] spambayes/spambayes ImapUI.py, 1.18,
	1.18.2.1 ProxyUI.py, 1.23, 1.23.2.1 UserInterface.py, 1.24,
	1.24.2.1 storage.py, 1.34, 1.34.2.1
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv16488/spambayes

Modified Files:
      Tag: release_1_0
	ImapUI.py ProxyUI.py UserInterface.py storage.py 
Log Message:
Stupid global variables!

Thanks to a global variable not being updated, when we recreated everything, the userinterface kept using the old classifier.
Since we now behave and close that one, this caused all sorts of problems.  Get rid of the damn global variable, and correctly
update it, and all is well in the world again.

In addition, don't save an empty database.  I think we make assumptions about the db being non-empty in some places.

This should fix [ spambayes-Bugs-809769 ] TypeError when training 1.0a6

(I can't believe it took so long for me to find this!)

Index: ImapUI.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/ImapUI.py,v
retrieving revision 1.18
retrieving revision 1.18.2.1
diff -C2 -d -r1.18 -r1.18.2.1
*** ImapUI.py	18 Sep 2003 03:58:59 -0000	1.18
--- ImapUI.py	24 Sep 2003 03:54:15 -0000	1.18.2.1
***************
*** 47,52 ****
  from Options import options, optionsPathname
  
- global classifier
- 
  # These are the options that will be offered on the configuration page.
  # If the option is None, then the entry is a header and the following
--- 47,50 ----
***************
*** 108,112 ****
      """Serves the HTML user interface for the proxies."""
      def __init__(self, cls, imap, pwd):
!         global classifier, parm_map
          # Only offer SSL if it is available
          try:
--- 106,110 ----
      """Serves the HTML user interface for the proxies."""
      def __init__(self, cls, imap, pwd):
!         global parm_map
          # Only offer SSL if it is available
          try:
***************
*** 117,121 ****
              parm_map = tuple(parm_list)
          UserInterface.UserInterface.__init__(self, cls, parm_map, adv_map)
!         classifier = cls
          self.imap = imap
          self.imap_pwd = pwd
--- 115,119 ----
              parm_map = tuple(parm_list)
          UserInterface.UserInterface.__init__(self, cls, parm_map, adv_map)
!         self.classifier = cls
          self.imap = imap
          self.imap_pwd = pwd
***************
*** 125,130 ****
      def onHome(self):
          """Serve up the homepage."""
!         stateDict = classifier.__dict__.copy()
!         stateDict.update(classifier.__dict__)
          statusTable = self.html.statusTable.clone()
          del statusTable.proxyDetails
--- 123,128 ----
      def onHome(self):
          """Serve up the homepage."""
!         stateDict = self.classifier.__dict__.copy()
!         stateDict.update(self.classifier.__dict__)
          statusTable = self.html.statusTable.clone()
          del statusTable.proxyDetails
***************
*** 149,154 ****
          restores the defaults."""
          # Reload the options.
!         global classifier
!         classifier.store()
          import Options
          reload(Options)
--- 147,151 ----
          restores the defaults."""
          # Reload the options.
!         self.classifier.store()
          import Options
          reload(Options)

Index: ProxyUI.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/ProxyUI.py,v
retrieving revision 1.23
retrieving revision 1.23.2.1
diff -C2 -d -r1.23 -r1.23.2.1
*** ProxyUI.py	18 Sep 2003 13:55:11 -0000	1.23
--- ProxyUI.py	24 Sep 2003 03:54:15 -0000	1.23.2.1
***************
*** 511,515 ****
          # Reload the options.
          global state
-         state.bayes.store()
          import Options
          reload(Options)
--- 511,514 ----
***************
*** 519,522 ****
--- 518,522 ----
          # Recreate the state.
          state = self.state_recreator()
+         self.classifier = state.bayes
  
      def verifyInput(self, parms, pmap):

Index: UserInterface.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/UserInterface.py,v
retrieving revision 1.24
retrieving revision 1.24.2.1
diff -C2 -d -r1.24 -r1.24.2.1
*** UserInterface.py	8 Sep 2003 07:04:17 -0000	1.24
--- UserInterface.py	24 Sep 2003 03:54:15 -0000	1.24.2.1
***************
*** 86,91 ****
            'message', 'train', 'classify', 'query')
  
- global classifier
- 
  class UserInterfaceServer(Dibbler.HTTPServer):
      """Implements the web server component via a Dibbler plugin."""
--- 86,89 ----
***************
*** 246,252 ****
      def __init__(self, bayes, config_parms=(), adv_parms=()):
          """Load up the necessary resources: ui.html and helmet.gif."""
-         global classifier
          BaseUserInterface.__init__(self)
!         classifier = bayes
          self.parm_ini_map = config_parms
          self.advanced_options_map = adv_parms
--- 244,249 ----
      def __init__(self, bayes, config_parms=(), adv_parms=()):
          """Load up the necessary resources: ui.html and helmet.gif."""
          BaseUserInterface.__init__(self)
!         self.classifier = bayes
          self.parm_ini_map = config_parms
          self.advanced_options_map = adv_parms
***************
*** 266,270 ****
          cluesRow = cluesTable.cluesRow.clone()
          del cluesTable.cluesRow   # Delete dummy row to make way for real ones
!         (probability, clues) = classifier.spamprob(tokenizer.tokenize(message),\
                                                      evidence=True)
          for word, wordProb in clues:
--- 263,267 ----
          cluesRow = cluesTable.cluesRow.clone()
          del cluesTable.cluesRow   # Delete dummy row to make way for real ones
!         (probability, clues) = self.classifier.spamprob(tokenizer.tokenize(message),\
                                                      evidence=True)
          for word, wordProb in clues:
***************
*** 294,298 ****
                  word = word[:-1]
                  reached_limit = False
!                 for w in classifier._wordinfokeys():
                      if not reached_limit and len(statsBoxes) > wildcard_limit:
                          reached_limit = True
--- 291,295 ----
                  word = word[:-1]
                  reached_limit = False
!                 for w in self.classifier._wordinfokeys():
                      if not reached_limit and len(statsBoxes) > wildcard_limit:
                          reached_limit = True
***************
*** 302,310 ****
                              over_limit += 1
                          else:
!                             wordinfo = classifier._wordinfoget(w)
                              stats = self.html.wordStats.clone()
                              stats.spamcount = wordinfo.spamcount
                              stats.hamcount = wordinfo.hamcount
!                             stats.spamprob = classifier.probability(wordinfo)
                              box = self._buildBox("Statistics for %r" % \
                                                   cgi.escape(w),
--- 299,307 ----
                              over_limit += 1
                          else:
!                             wordinfo = self.classifier._wordinfoget(w)
                              stats = self.html.wordStats.clone()
                              stats.spamcount = wordinfo.spamcount
                              stats.hamcount = wordinfo.hamcount
!                             stats.spamprob = self.classifier.probability(wordinfo)
                              box = self._buildBox("Statistics for %r" % \
                                                   cgi.escape(w),
***************
*** 334,343 ****
              else:
                  # Optimised version for non-wildcard searches
!                 wordinfo = classifier._wordinfoget(word)
                  if wordinfo:
                      stats = self.html.wordStats.clone()
                      stats.spamcount = wordinfo.spamcount
                      stats.hamcount = wordinfo.hamcount
!                     stats.spamprob = classifier.probability(wordinfo)
                  else:
                      stats = "%r does not exist in the database." % cgi.escape(word)
--- 331,340 ----
              else:
                  # Optimised version for non-wildcard searches
!                 wordinfo = self.classifier._wordinfoget(word)
                  if wordinfo:
                      stats = self.html.wordStats.clone()
                      stats.spamcount = wordinfo.spamcount
                      stats.hamcount = wordinfo.hamcount
!                     stats.spamprob = self.classifier.probability(wordinfo)
                  else:
                      stats = "%r does not exist in the database." % cgi.escape(word)
***************
*** 386,390 ****
          for message in messages:
              tokens = tokenizer.tokenize(message)
!             classifier.learn(tokens, isSpam)
              f.write("From pop3proxy@spambayes.org Sat Jan 31 00:00:00 2000\n")
              f.write(message)
--- 383,387 ----
          for message in messages:
              tokens = tokenizer.tokenize(message)
!             self.classifier.learn(tokens, isSpam)
              f.write("From pop3proxy@spambayes.org Sat Jan 31 00:00:00 2000\n")
              f.write(message)
***************
*** 468,472 ****
          self.write("Saving... ")
          self.flush()
!         classifier.store()
          self.write("Done.\n")
  
--- 465,469 ----
          self.write("Saving... ")
          self.flush()
!         self.classifier.store()
          self.write("Done.\n")
  

Index: storage.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v
retrieving revision 1.34
retrieving revision 1.34.2.1
diff -C2 -d -r1.34 -r1.34.2.1
*** storage.py	18 Sep 2003 14:04:31 -0000	1.34
--- storage.py	24 Sep 2003 03:54:15 -0000	1.34.2.1
***************
*** 169,172 ****
--- 169,181 ----
          getattr(self.dbm, "close", noop)()
          # should not be a need to drop the 'dbm' or 'db' attributes.
+         # but we do anyway, because it makes it more clear what has gone
+         # wrong if we try to keep using the database after we have closed
+         # it.
+         if hasattr(self, "db"):
+             del self.db
+         if hasattr(self, "dbm"):
+             del self.dbm
+         if options["globals", "verbose"]:
+             print >> sys.stderr, 'Closed',self.db_name,'database'
  
      def load(self):



From anadelonbrin at users.sourceforge.net  Wed Sep 24 01:28:55 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Wed Sep 24 01:28:59 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_server.py,1.7,1.8
Message-ID: 

Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv32618/scripts

Modified Files:
	sb_server.py 
Log Message:
This should fix [ spambayes-Bugs-809769 ] TypeError when training 1.0a6

Index: sb_server.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_server.py,v
retrieving revision 1.7
retrieving revision 1.8
diff -C2 -d -r1.7 -r1.8
*** sb_server.py	19 Sep 2003 23:38:10 -0000	1.7
--- sb_server.py	24 Sep 2003 05:28:53 -0000	1.8
***************
*** 724,728 ****
      # complains if we try to reopen it without closing it first.
      if hasattr(state, "bayes"):
!         state.bayes.store()
          state.bayes.close()
  
--- 724,730 ----
      # complains if we try to reopen it without closing it first.
      if hasattr(state, "bayes"):
!         # Only store a non-empty db.
!         if state.bayes.nham != 0 and state.bayes.nspam != 0:
!             state.bayes.store()
          state.bayes.close()
  



From anadelonbrin at users.sourceforge.net  Wed Sep 24 01:28:56 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Wed Sep 24 01:29:03 2003
Subject: [Spambayes-checkins] spambayes/spambayes ImapUI.py, 1.19,
	1.20 ProxyUI.py, 1.25, 1.26 ServerUI.py, 1.1,
	1.2 UserInterface.py, 1.25, 1.26 storage.py, 1.34, 1.35
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv32618/spambayes

Modified Files:
	ImapUI.py ProxyUI.py ServerUI.py UserInterface.py storage.py 
Log Message:
This should fix [ spambayes-Bugs-809769 ] TypeError when training 1.0a6

Index: ImapUI.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/ImapUI.py,v
retrieving revision 1.19
retrieving revision 1.20
diff -C2 -d -r1.19 -r1.20
*** ImapUI.py	19 Sep 2003 23:38:10 -0000	1.19
--- ImapUI.py	24 Sep 2003 05:28:53 -0000	1.20
***************
*** 47,52 ****
  from Options import options, optionsPathname
  
- global classifier
- 
  # These are the options that will be offered on the configuration page.
  # If the option is None, then the entry is a header and the following
--- 47,50 ----
***************
*** 110,114 ****
      """Serves the HTML user interface for the proxies."""
      def __init__(self, cls, imap, pwd):
!         global classifier, parm_map
          # Only offer SSL if it is available
          try:
--- 108,112 ----
      """Serves the HTML user interface for the proxies."""
      def __init__(self, cls, imap, pwd):
!         global parm_map
          # Only offer SSL if it is available
          try:
***************
*** 119,123 ****
              parm_map = tuple(parm_list)
          UserInterface.UserInterface.__init__(self, cls, parm_map, adv_map)
!         classifier = cls
          self.imap = imap
          self.imap_pwd = pwd
--- 117,121 ----
              parm_map = tuple(parm_list)
          UserInterface.UserInterface.__init__(self, cls, parm_map, adv_map)
!         self.classifier = cls
          self.imap = imap
          self.imap_pwd = pwd
***************
*** 127,132 ****
      def onHome(self):
          """Serve up the homepage."""
!         stateDict = classifier.__dict__.copy()
!         stateDict.update(classifier.__dict__)
          statusTable = self.html.statusTable.clone()
          del statusTable.proxyDetails
--- 125,130 ----
      def onHome(self):
          """Serve up the homepage."""
!         stateDict = self.classifier.__dict__.copy()
!         stateDict.update(self.classifier.__dict__)
          statusTable = self.html.statusTable.clone()
          del statusTable.proxyDetails
***************
*** 154,159 ****
          restores the defaults."""
          # Reload the options.
!         global classifier
!         classifier.store()
          import Options
          reload(Options)
--- 152,156 ----
          restores the defaults."""
          # Reload the options.
!         self.classifier.store()
          import Options
          reload(Options)

Index: ProxyUI.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/ProxyUI.py,v
retrieving revision 1.25
retrieving revision 1.26
diff -C2 -d -r1.25 -r1.26
*** ProxyUI.py	19 Sep 2003 23:47:11 -0000	1.25
--- ProxyUI.py	24 Sep 2003 05:28:53 -0000	1.26
***************
*** 660,664 ****
          # Reload the options.
          global state
-         state.bayes.store()
          import Options
          reload(Options)
--- 660,663 ----
***************
*** 668,671 ****
--- 667,671 ----
          # Recreate the state.
          state = self.state_recreator()
+         self.classifier = state.bayes
  
      def verifyInput(self, parms, pmap):

Index: ServerUI.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/ServerUI.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** ServerUI.py	6 Sep 2003 04:03:35 -0000	1.1
--- ServerUI.py	24 Sep 2003 05:28:53 -0000	1.2
***************
*** 101,105 ****
          self.state = self.state_recreator()
  
!     def verifyInput(self, parms):
          '''Check that the given input is valid.'''
          # Most of the work here is done by the parent class, but
--- 101,105 ----
          self.state = self.state_recreator()
  
!     def verifyInput(self, parms, pmap):
          '''Check that the given input is valid.'''
          # Most of the work here is done by the parent class, but

Index: UserInterface.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/UserInterface.py,v
retrieving revision 1.25
retrieving revision 1.26
diff -C2 -d -r1.25 -r1.26
*** UserInterface.py	19 Sep 2003 23:38:10 -0000	1.25
--- UserInterface.py	24 Sep 2003 05:28:53 -0000	1.26
***************
*** 86,91 ****
            'message', 'train', 'classify', 'query')
  
- global classifier
- 
  class UserInterfaceServer(Dibbler.HTTPServer):
      """Implements the web server component via a Dibbler plugin."""
--- 86,89 ----
***************
*** 246,252 ****
      def __init__(self, bayes, config_parms=(), adv_parms=()):
          """Load up the necessary resources: ui.html and helmet.gif."""
-         global classifier
          BaseUserInterface.__init__(self)
!         classifier = bayes
          self.parm_ini_map = config_parms
          self.advanced_options_map = adv_parms
--- 244,249 ----
      def __init__(self, bayes, config_parms=(), adv_parms=()):
          """Load up the necessary resources: ui.html and helmet.gif."""
          BaseUserInterface.__init__(self)
!         self.classifier = bayes
          self.parm_ini_map = config_parms
          self.advanced_options_map = adv_parms
***************
*** 274,278 ****
          cluesRow = cluesTable.cluesRow.clone()
          del cluesTable.cluesRow   # Delete dummy row to make way for real ones
!         fetchword = classifier._wordinfoget
          for word, wordProb in clues:
              record = fetchword(word)
--- 271,275 ----
          cluesRow = cluesTable.cluesRow.clone()
          del cluesTable.cluesRow   # Delete dummy row to make way for real ones
!         fetchword = self.classifier._wordinfoget
          for word, wordProb in clues:
              record = fetchword(word)
***************
*** 281,285 ****
                  nspam = record.spamcount
                  if wordProb is None:
!                     wordProb = classifier.probability(record)
              elif word != "*H*" and word != "*S*":
                  nham = nspam = 0
--- 278,282 ----
                  nspam = record.spamcount
                  if wordProb is None:
!                     wordProb = self.classifier.probability(record)
              elif word != "*H*" and word != "*S*":
                  nham = nspam = 0
***************
*** 300,308 ****
              for tok in tokens:
                  clues.append((tok, None))
!             probability = classifier.spamprob(tokens)
              cluesTable = self._fillCluesTable(clues)
              head_name = "Tokens"
          else:
!             (probability, clues) = classifier.spamprob(tokens, evidence=True)
              cluesTable = self._fillCluesTable(clues)
              head_name = "Clues"
--- 297,305 ----
              for tok in tokens:
                  clues.append((tok, None))
!             probability = self.classifier.spamprob(tokens)
              cluesTable = self._fillCluesTable(clues)
              head_name = "Tokens"
          else:
!             (probability, clues) = self.classifier.spamprob(tokens, evidence=True)
              cluesTable = self._fillCluesTable(clues)
              head_name = "Clues"
***************
*** 379,386 ****
              stats.append("You must enter a word.")
          elif query_type == "basic" and not ignore_case:
!             wordinfo = classifier._wordinfoget(word)
              if wordinfo:
                  stat = (word, wordinfo.spamcount, wordinfo.hamcount,
!                         classifier.probability(wordinfo))
              else:
                  stat = "%r does not exist in the database." % \
--- 376,383 ----
              stats.append("You must enter a word.")
          elif query_type == "basic" and not ignore_case:
!             wordinfo = self.classifier._wordinfoget(word)
              if wordinfo:
                  stat = (word, wordinfo.spamcount, wordinfo.hamcount,
!                         self.classifier.probability(wordinfo))
              else:
                  stat = "%r does not exist in the database." % \
***************
*** 400,404 ****
  
              reached_limit = False
!             for w in classifier._wordinfokeys():
                  if not reached_limit and len(stats) >= max_results:
                      reached_limit = True
--- 397,401 ----
  
              reached_limit = False
!             for w in self.classifier._wordinfokeys():
                  if not reached_limit and len(stats) >= max_results:
                      reached_limit = True
***************
*** 408,414 ****
                          over_limit += 1
                      else:
!                         wordinfo = classifier._wordinfoget(w)
                          stat = (w, wordinfo.spamcount, wordinfo.hamcount,
!                                 classifier.probability(wordinfo))
                          stats.append(stat)
              if len(stats) == 0 and max_results > 0:
--- 405,411 ----
                          over_limit += 1
                      else:
!                         wordinfo = self.classifier._wordinfoget(w)
                          stat = (w, wordinfo.spamcount, wordinfo.hamcount,
!                                 self.classifier.probability(wordinfo))
                          stats.append(stat)
              if len(stats) == 0 and max_results > 0:
***************
*** 495,499 ****
          for message in messages:
              tokens = tokenizer.tokenize(message)
!             classifier.learn(tokens, isSpam)
              f.write("From pop3proxy@spambayes.org Sat Jan 31 00:00:00 2000\n")
              f.write(message)
--- 492,496 ----
          for message in messages:
              tokens = tokenizer.tokenize(message)
!             self.classifier.learn(tokens, isSpam)
              f.write("From pop3proxy@spambayes.org Sat Jan 31 00:00:00 2000\n")
              f.write(message)
***************
*** 577,581 ****
          self.write("Saving... ")
          self.flush()
!         classifier.store()
          self.write("Done.\n")
  
--- 574,578 ----
          self.write("Saving... ")
          self.flush()
!         self.classifier.store()
          self.write("Done.\n")
  

Index: storage.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v
retrieving revision 1.34
retrieving revision 1.35
diff -C2 -d -r1.34 -r1.35
*** storage.py	18 Sep 2003 14:04:31 -0000	1.34
--- storage.py	24 Sep 2003 05:28:53 -0000	1.35
***************
*** 169,172 ****
--- 169,181 ----
          getattr(self.dbm, "close", noop)()
          # should not be a need to drop the 'dbm' or 'db' attributes.
+         # but we do anyway, because it makes it more clear what has gone
+         # wrong if we try to keep using the database after we have closed
+         # it.
+         if hasattr(self, "db"):
+             del self.db
+         if hasattr(self, "dbm"):
+             del self.dbm
+         if options["globals", "verbose"]:
+             print >> sys.stderr, 'Closed',self.db_name,'database'
  
      def load(self):



From anadelonbrin at users.sourceforge.net  Wed Sep 24 02:16:53 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Wed Sep 24 02:17:47 2003
Subject: [Spambayes-checkins] website server_side.ht,1.1,1.2
Message-ID: 

Update of /cvsroot/spambayes/website
In directory sc8-pr-cvs1:/tmp/cvs-serv7793

Modified Files:
	server_side.ht 
Log Message:
Add qmail notes from Michael Martinez

Index: server_side.ht
===================================================================
RCS file: /cvsroot/spambayes/website/server_side.ht,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** server_side.ht	29 Jul 2003 08:36:15 -0000	1.1
--- server_side.ht	24 Sep 2003 06:16:51 -0000	1.2
***************
*** 40,41 ****
--- 40,122 ----
  
  
+ 
+ 
                qmail notes from Michael Martinez
                
+ 
                SpamBayes is installed on our agency's smtp / MX gateway. This machine runs Redhat
+ Linux 7.1, qmail 1.03, qmail-scanner 1.16, and hbedv's Antivir. Incoming mail is
+ accepted by tcpserver and handed off to qmail-scanner. Qmail-scanner runs the virus
+ software (antivir) and hands the message to qmail. Qmail accepts local delivery
+ on all domain-bound email. This email is delivered to ~alias/.qmail-default.
+ (This is a standard configuration for qmail).
                
+ 
+ 
                ~alias/.qmail-default pipes each email through Spambayes. The .qmail-default
+ is set up as follows:
                
+ 
+ 
                + | /usr/local/spambayes/hammiefilter.py -d /usr/local/spambayes/.hammiedb | qmail-remote MSServer.csrees.usda.gov "$SENDER" $DEFAULT@csrees.usda.gov
+ 
                
+ 
                
+  
+ 
                The permissions for the /usr/local/spambayes directory are set with the following command:
                
+ 
                chown R qmailq.qmail /usr/local/spambayes
                
+ 
                 
+ 
+ 
                As shown above, there are two pipes. The first pipes it through Spambayes.
+ The second pipes it through qmail's remote delivery mechanism, which delivers
+ the email to our Exchange Server.
                
+ 
+ 
                Delivered emails are filtered on a per-user basis in Outlook by setting
+ the Rules to detect the Spambayes tag in the message header. If the tag
+ reads Spambayes-Classification: spam then the email is either deleted
+ or placed in the user's Spam folder. If it reads Spambayes-Classification: unsure
+ then it's placed in the user's Unsure folder. If it reads Spambayes-Classification: ham
+ then nothing special is done  it is delivered to the user's Inbox as normal.
                
+  
+ 
                The user is given the choice of whether to set up his rules or not.
                
+ 
+ 
                Training of Spambayes is done in the following manner: our users are
+ given my email address and are told that, if they like, they may send
+ emails to me that they consider spam, or that end up being mis-classified
+ by the system. I created two directories:
                
+ 
+ 
                + /usr/local/spambayes/training/spamdir
+ /usr/local/spambayes/training/hamdir
+ 
                
+ 
                
+  
+ 
                The emails sent to me by the users are retrieved from the qmail archive
+ and placed into the appropriate directory.  When I'm ready to do a training
+ (which I do once or twice a month), I run the following commands: 
                
+ 
+ 
                  
+ 
                1. I use a simple script to insert a blank From: line at the top of each email
                  2. 
+ 
                2. I use a simple script to remove the qmail-scanner header from the bottom of each email.
                  3. 
+ 
                3. uuencoded attachments are removed
                  4. 
+ 
                4. cat /usr/local/spambayes/training/spamdir/* >> /usr/local/spambayes/training/spam
                  5. 
+ 
                5. cat /usr/local/spambayes/training/hamdir/* >> /usr/local/spambayes/training/ham 
                  6. 
+ 
                6. /usr/local/spambayes/mboxtrain d /usr/local/spambayes/.hammiedb g /usr/local/spambayes/training/ham s /usr/local/spambayes/training/spam
                  
+ (This last step can be run without shutting down qmail.)
                  7. 
+ 
                
+ 
                
+ 
+ 
                Most of the time, emails that are sent to me are clearly discernible as
+ to whether they are spam or not. Occasionally there is an email that is
+ borderline, or that one person considers spam but others don't. This is
+ usually things like newsletter subscriptions or religious forums. In this
+ case, I follow my own rule that if there is at least one person in the
+ agency who needs or wants to receive this type of email, and as long as it
+ is non-offensive, work-related, or there are a lot of people in the agency
+ who have an interest in the topic, then I will either train it as ham, or,
+ if it's already being tagged ham, leave it. An example of this are emails
+ that discuss religious topics. There are a lot of people in this agency who
+ are subscribed to religious discussion groups, so in my mind, it's good
+ practice to make sure these messages are not tagged spam.
                
+ 
+ 
                The above system works well on several levels. It's manageable because
+ there's a central location for training and tagging spam (the smtp server).
+ It's manageable also because our IT PC Support staff does not have to install
+ SpamBayes on each PC nor train all of our users on its use. If a user does
+ not like the way our system tags the emails, he does not have to set up his
+ Outlook rules. But, we've had a good response from the users who are using
+ their Rules. They're willing to put up with one or two mis-classified emails
+ in order to have 95% of their junk email not in their Inbox.
                



From anadelonbrin at users.sourceforge.net  Wed Sep 24 02:38:32 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Wed Sep 24 02:38:35 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_xmlrpcserver.py,1.1,1.2
Message-ID: 

Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv11384/scripts

Modified Files:
	sb_xmlrpcserver.py 
Log Message:
Bring sb_xmlrpcserver into line with the rest of the spambayes world.

This is pretty much the diff that Skip posted to the list, with a couple
of minor changes.  Apparently it didn't work for him, but it works fine
for me (with sb_client.py).

Index: sb_xmlrpcserver.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_xmlrpcserver.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** sb_xmlrpcserver.py	5 Sep 2003 01:16:46 -0000	1.1
--- sb_xmlrpcserver.py	24 Sep 2003 06:38:30 -0000	1.2
***************
*** 11,19 ****
      -p FILE
          use file as the persistent store.  loads data from this file if it
!         exists, and saves data to this file at the end.  Default: %(DEFAULTDB)s
      -d
!         use the DBM store instead of cPickle.  The file is larger and
!         creating it is slower, but checking against it is much faster,
!         especially for large word databases.
  
      IP
--- 11,17 ----
      -p FILE
          use file as the persistent store.  loads data from this file if it
!         exists, and saves data to this file at the end.
      -d
!         use the DBM store instead of cPickle.
  
      IP
***************
*** 23,26 ****
--- 21,25 ----
  """
  
+ import os
  import SimpleXMLRPCServer
  import getopt
***************
*** 28,32 ****
  import traceback
  import xmlrpclib
! from spambayes import hammie
  
  try:
--- 27,33 ----
  import traceback
  import xmlrpclib
! 
! from spambayes import hammie, Options
! from spambayes.storage import open_storage
  
  try:
***************
*** 39,45 ****
  program = sys.argv[0] # For usage(); referenced by docstring above
  
- # Default DB path
- DEFAULTDB = hammie.DEFAULTDB
- 
  class XMLHammie(hammie.Hammie):
      def score(self, msg, *extra):
--- 40,43 ----
***************
*** 58,108 ****
  
  
- class HammieHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler):
-     def do_POST(self):
-         """Handles the HTTP POST request.
- 
-         Attempts to interpret all HTTP POST requests as XML-RPC calls,
-         which are forwarded to the _dispatch method for handling.
- 
-         This one also prints out tracebacks, to help me debug :)
-         """
- 
-         try:
-             # get arguments
-             data = self.rfile.read(int(self.headers["content-length"]))
-             params, method = xmlrpclib.loads(data)
- 
-             # generate response
-             try:
-                 response = self._dispatch(method, params)
-                 # wrap response in a singleton tuple
-                 response = (response,)
-             except:
-                 traceback.print_exc()
-                 # report exception back to server
-                 response = xmlrpclib.dumps(
-                     xmlrpclib.Fault(1, "%s:%s" % (sys.exc_type, sys.exc_value))
-                     )
-             else:
-                 response = xmlrpclib.dumps(response, methodresponse=1)
-         except:
-             # internal error, report as HTTP server error
-             traceback.print_exc()
-             print `data`
-             self.send_response(500)
-             self.end_headers()
-         else:
-             # got a valid XML RPC response
-             self.send_response(200)
-             self.send_header("Content-type", "text/xml")
-             self.send_header("Content-length", str(len(response)))
-             self.end_headers()
-             self.wfile.write(response)
- 
-             # shut down the connection
-             self.wfile.flush()
-             self.connection.shutdown(1)
- 
- 
  def usage(code, msg=''):
      """Print usage message and sys.exit(code)."""
--- 56,59 ----
***************
*** 110,114 ****
          print >> sys.stderr, msg
          print >> sys.stderr
!     print >> sys.stderr, __doc__ % globals()
      sys.exit(code)
  
--- 61,65 ----
          print >> sys.stderr, msg
          print >> sys.stderr
!     print >> sys.stderr, __doc__
      sys.exit(code)
  
***************
*** 121,131 ****
          usage(2, msg)
  
!     pck = DEFAULTDB
!     usedb = False
      for opt, arg in opts:
          if opt == '-h':
              usage(0)
          elif opt == '-p':
!             pck = arg
          elif opt == "-d":
              usedb = True
--- 72,85 ----
          usage(2, msg)
  
!     options = Options.options
! 
!     dbname = options["Storage", "persistent_storage_file"]
!     dbname = os.path.expanduser(dbname)
!     usedb = options["Storage", "persistent_use_database"]
      for opt, arg in opts:
          if opt == '-h':
              usage(0)
          elif opt == '-p':
!             dbname = arg
          elif opt == "-d":
              usedb = True
***************
*** 137,144 ****
      port = int(port)
  
!     bayes = hammie.createbayes(pck, usedb)
      h = XMLHammie(bayes)
  
!     server = SimpleXMLRPCServer.SimpleXMLRPCServer((ip, port), HammieHandler)
      server.register_instance(h)
      server.serve_forever()
--- 91,99 ----
      port = int(port)
  
!     bayes = open_storage(dbname, usedb)
      h = XMLHammie(bayes)
  
!     server = SimpleXMLRPCServer.SimpleXMLRPCServer((ip, port),
!                                                    SimpleXMLRPCServer.SimpleXMLRPCRequestHandler)
      server.register_instance(h)
      server.serve_forever()



From anadelonbrin at users.sourceforge.net  Wed Sep 24 02:39:21 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Wed Sep 24 02:39:29 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_xmlrpcserver.py, 1.1,
	1.1.2.1
Message-ID: 

Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv11527/scripts

Modified Files:
      Tag: release_1_0
	sb_xmlrpcserver.py 
Log Message:
Bring sb_xmlrpcserver into line with the rest of the spambayes world.

This is pretty much the diff that Skip posted to the list, with a couple
of minor changes.  Apparently it didn't work for him, but it works fine
for me (with sb_client.py).

Index: sb_xmlrpcserver.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_xmlrpcserver.py,v
retrieving revision 1.1
retrieving revision 1.1.2.1
diff -C2 -d -r1.1 -r1.1.2.1
*** sb_xmlrpcserver.py	5 Sep 2003 01:16:46 -0000	1.1
--- sb_xmlrpcserver.py	24 Sep 2003 06:39:19 -0000	1.1.2.1
***************
*** 11,19 ****
      -p FILE
          use file as the persistent store.  loads data from this file if it
!         exists, and saves data to this file at the end.  Default: %(DEFAULTDB)s
      -d
!         use the DBM store instead of cPickle.  The file is larger and
!         creating it is slower, but checking against it is much faster,
!         especially for large word databases.
  
      IP
--- 11,17 ----
      -p FILE
          use file as the persistent store.  loads data from this file if it
!         exists, and saves data to this file at the end.
      -d
!         use the DBM store instead of cPickle.
  
      IP
***************
*** 23,26 ****
--- 21,25 ----
  """
  
+ import os
  import SimpleXMLRPCServer
  import getopt
***************
*** 28,32 ****
  import traceback
  import xmlrpclib
! from spambayes import hammie
  
  try:
--- 27,33 ----
  import traceback
  import xmlrpclib
! 
! from spambayes import hammie, Options
! from spambayes.storage import open_storage
  
  try:
***************
*** 39,45 ****
  program = sys.argv[0] # For usage(); referenced by docstring above
  
- # Default DB path
- DEFAULTDB = hammie.DEFAULTDB
- 
  class XMLHammie(hammie.Hammie):
      def score(self, msg, *extra):
--- 40,43 ----
***************
*** 58,108 ****
  
  
- class HammieHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler):
-     def do_POST(self):
-         """Handles the HTTP POST request.
- 
-         Attempts to interpret all HTTP POST requests as XML-RPC calls,
-         which are forwarded to the _dispatch method for handling.
- 
-         This one also prints out tracebacks, to help me debug :)
-         """
- 
-         try:
-             # get arguments
-             data = self.rfile.read(int(self.headers["content-length"]))
-             params, method = xmlrpclib.loads(data)
- 
-             # generate response
-             try:
-                 response = self._dispatch(method, params)
-                 # wrap response in a singleton tuple
-                 response = (response,)
-             except:
-                 traceback.print_exc()
-                 # report exception back to server
-                 response = xmlrpclib.dumps(
-                     xmlrpclib.Fault(1, "%s:%s" % (sys.exc_type, sys.exc_value))
-                     )
-             else:
-                 response = xmlrpclib.dumps(response, methodresponse=1)
-         except:
-             # internal error, report as HTTP server error
-             traceback.print_exc()
-             print `data`
-             self.send_response(500)
-             self.end_headers()
-         else:
-             # got a valid XML RPC response
-             self.send_response(200)
-             self.send_header("Content-type", "text/xml")
-             self.send_header("Content-length", str(len(response)))
-             self.end_headers()
-             self.wfile.write(response)
- 
-             # shut down the connection
-             self.wfile.flush()
-             self.connection.shutdown(1)
- 
- 
  def usage(code, msg=''):
      """Print usage message and sys.exit(code)."""
--- 56,59 ----
***************
*** 110,114 ****
          print >> sys.stderr, msg
          print >> sys.stderr
!     print >> sys.stderr, __doc__ % globals()
      sys.exit(code)
  
--- 61,65 ----
          print >> sys.stderr, msg
          print >> sys.stderr
!     print >> sys.stderr, __doc__
      sys.exit(code)
  
***************
*** 121,131 ****
          usage(2, msg)
  
!     pck = DEFAULTDB
!     usedb = False
      for opt, arg in opts:
          if opt == '-h':
              usage(0)
          elif opt == '-p':
!             pck = arg
          elif opt == "-d":
              usedb = True
--- 72,85 ----
          usage(2, msg)
  
!     options = Options.options
! 
!     dbname = options["Storage", "persistent_storage_file"]
!     dbname = os.path.expanduser(dbname)
!     usedb = options["Storage", "persistent_use_database"]
      for opt, arg in opts:
          if opt == '-h':
              usage(0)
          elif opt == '-p':
!             dbname = arg
          elif opt == "-d":
              usedb = True
***************
*** 137,144 ****
      port = int(port)
  
!     bayes = hammie.createbayes(pck, usedb)
      h = XMLHammie(bayes)
  
!     server = SimpleXMLRPCServer.SimpleXMLRPCServer((ip, port), HammieHandler)
      server.register_instance(h)
      server.serve_forever()
--- 91,99 ----
      port = int(port)
  
!     bayes = open_storage(dbname, usedb)
      h = XMLHammie(bayes)
  
!     server = SimpleXMLRPCServer.SimpleXMLRPCServer((ip, port),
!                                                    SimpleXMLRPCServer.SimpleXMLRPCRequestHandler)
      server.register_instance(h)
      server.serve_forever()



From anadelonbrin at users.sourceforge.net  Wed Sep 24 02:53:39 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Wed Sep 24 02:53:42 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_filter.py,1.2,1.3
Message-ID: 

Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv13444/scripts

Modified Files:
	sb_filter.py 
Log Message:
If the -n switch was before the -d/-p switch, then the name wouldn't be used.
This is rather unintuitive, so fix this so that the -d/-p name is used wherever
the -n switch is.

Index: sb_filter.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_filter.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** sb_filter.py	10 Sep 2003 04:33:17 -0000	1.2
--- sb_filter.py	24 Sep 2003 06:53:36 -0000	1.3
***************
*** 53,56 ****
--- 53,62 ----
  from spambayes import hammie, Options, mboxutils
  
+ try:
+     True, False
+ except NameError:
+     # Maintain compatibility with Python 2.2
+     True, False = 1, 0
+ 
  # See Options.py for explanations of these properties
  program = sys.argv[0]
***************
*** 148,151 ****
--- 154,158 ----
      opts, args = getopt.getopt(sys.argv[1:], 'hxd:D:nfgstGS',
                                 ['help', 'examples'])
+     create_newdb = False
      for opt, arg in opts:
          if opt in ('-h', '--help'):
***************
*** 172,177 ****
              actions.append(h.untrain_spam)
          elif opt == "-n":
!             h.newdb()
!             sys.exit(0)
  
      if actions == []:
--- 179,187 ----
              actions.append(h.untrain_spam)
          elif opt == "-n":
!             create_newdb = True
! 
!     if create_newdb:
!         h.newdb()
!         sys.exit(0)
  
      if actions == []:



From anadelonbrin at users.sourceforge.net  Wed Sep 24 02:53:39 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Wed Sep 24 02:53:45 2003
Subject: [Spambayes-checkins] spambayes/spambayes/test test_smtpproxy.py,
	NONE, 1.1
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes/test
In directory sc8-pr-cvs1:/tmp/cvs-serv13444/spambayes/test

Added Files:
	test_smtpproxy.py 
Log Message:
If the -n switch was before the -d/-p switch, then the name wouldn't be used.
This is rather unintuitive, so fix this so that the -d/-p name is used wherever
the -n switch is.

--- NEW FILE: test_smtpproxy.py ---
#! /usr/bin/env python

"""Test that the SMTP proxy is working correctly.

When using the -z command line option, carries out various tests.

The -t option runs a fake SMTP server on port 8025.  This is the
same server that the testing option uses, and may be separately run for
other testing purposes.

Usage:

    test_smtpproxy.py [options]

        options:
            -t      : Runs a fake SMTP server on port 8025 (for testing).
            -h      : Displays this help message.

Any other options runs this in the standard Python unittest form.
"""

# This module is part of the spambayes project, which is Copyright 2002-3
# The Python Software Foundation and is covered by the Python Software
# Foundation license.

__author__ = "Tony Meyer "
__credits__ = "Richie Hindle, Mark Hammond, all the SpamBayes folk."

try:
    True, False
except NameError:
    # Maintain compatibility with Python 2.2
    True, False = 1, 0

# One example of spam and one of ham - both are used to train, and are
# then classified.  Not a good test of the classifier, but a perfectly
# good test of the SMTP proxy.  These are the same messages as in the
# POP3 proxy test (test_sb-server.py).

spam1 = """From: friend@public.com
Subject: Make money fast

Hello tim_chandler , Want to save money ?
Now is a good time to consider refinancing. Rates are low so you can cut
your current payments and save money.

http://64.251.22.101/interest/index%38%30%300%2E%68t%6D

Take off list on site [s5]
"""

good1 = """From: chris@example.com
Subject: ZPT and DTML

Jean Jordaan wrote:
> 'Fraid so ;>  It contains a vintage dtml-calendar tag.
>   http://www.zope.org/Members/teyc/CalendarTag
>
> Hmm I think I see what you mean: one needn't manually pass on the
> namespace to a ZPT?

Yeah, Page Templates are a bit more clever, sadly, DTML methods aren't :-(

Chris
"""

import re
import socket
import getopt
import asyncore
import operator
import unittest
import threading
import smtplib

# We need to import sb_server, but it may not be on the PYTHONPATH.
# Hack around this, so that if we are running in a cvs-like setup
# everything still works.
import os
import sys
try:
    this_file = __file__
except NameError:
    this_file = sys.argv[0]
sb_dir = os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(this_file))))
if sb_dir not in sys.path:
    sys.path.append(sb_dir)
    sys.path.append(os.path.join(sb_dir, "scripts"))

from spambayes import Dibbler
from spambayes import tokenizer
from spambayes.Options import options
from sb_server import state, _recreateState
from spambayes.smtpproxy import BayesSMTPProxyListener
from spambayes.ProxyUI import ProxyUserInterface
from spambayes.UserInterface import UserInterfaceServer

class TestListener(Dibbler.Listener):
    """Listener for TestPOP3Server.  Works on port 8025, because 8025
    wouldn't work for Tony."""

    def __init__(self, socketMap=asyncore.socket_map):
        Dibbler.Listener.__init__(self, 8025, TestSMTPServer,
                              (socketMap,), socketMap=socketMap)


class TestSMTPServer(Dibbler.BrighterAsyncChat):
    """Minimal SMTP server, for testing purposes.  Understands
    "MAIL FROM", "RCPT TO", "DATA" and "QUIT".  All mail is
    simply discarded. Also understands the 'KILL' command, to
    kill it."""

    def __init__(self, clientSocket, socketMap):
        # Grumble: asynchat.__init__ doesn't take a 'map' argument,
        # hence the two-stage construction.
        Dibbler.BrighterAsyncChat.__init__(self)
        Dibbler.BrighterAsyncChat.set_socket(self, clientSocket, socketMap)
        self.set_terminator('\r\n')
        self.okCommands = ['MAIL FROM:', 'RCPT TO:', 'DATA', 'QUIT', 'KILL',]
        self.handlers = {'MAIL FROM:': self.onFrom,
                         'RCPT TO:': self.onTo,
                         'DATA': self.onData,
                         'QUIT': self.onQuit,
                         'KILL': self.onKill,
                         }
        self.push("220 SpamBayes test SMTP server ready\r\n")
        self.request = ''
        self.inData = False

    def collect_incoming_data(self, data):
        """Asynchat override."""
        self.request = self.request + data
        print "data", data

    def push(self, data):
        print "pushing", repr(data)
        Dibbler.BrighterAsyncChat.push(self, data)
        
    def recv(self, buffer_size):
        """Asynchat override."""
        try:
            return Dibbler.BrighterAsyncChat.recv(self, buffer_size)
        except socket.error, e:
            if e[0] == 10053:
                return ''
            raise

    def found_terminator(self):
        """Asynchat override."""
        if self.inData:
            # Just throw the data away, unless it is the terminator.
            if self.request.strip() == '.':
                self.inData = False
                self.push("250 Message accepted for delivery\r\n")
        else:
            self.request = self.request.upper()
            foundCmd = False
            for cmd in self.okCommands:
                if self.request.startswith(cmd):
                    handler = self.handlers[cmd]
                    cooked = handler(self.request[len(cmd):])
                    if cooked is not None:
                        self.push(cooked.strip())
                    foundCmd = True
                    break
            if not foundCmd:
                # Something we don't know about.  Assume that it is ok!
                self.push("250 Unknown command ok.\r\n")
        self.request = ''

    def onKill(self, args):
        self.push("221 Goodbye\n") # Why not be polite 
        self.socket.shutdown(2)
        self.close()
        raise SystemExit

    def onQuit(self, args):
        self.push("221 Goodbye\r\n")
        self.close_when_done()

    def onFrom(self, args):
        # We don't care who it is from.
        return "250 %s... Sender ok\r\n" % (args.lower(),)

    def onTo(self, args):
        if args == options["smtpproxy", "ham_address"].upper():
            return "504 This command should not have got to the server\r\n"
        elif args == options["smtpproxy", "spam_address"].upper():
            return "504 This command should not have got to the server\r\n"
        return "250 %s... Recipient ok\r\n" % (args.lower(),)
    def onData(self, args):
        self.inData = True
        return '354 Enter mail, end with "." on a line by itself\r\n'


class SMTPProxyTest(unittest.TestCase):
    """Runs a self-test using TestSMTPServer, a minimal SMTP server
    that receives mail and discards it."""
    def setUp(self):
        # Run a proxy and a test server in separate threads with separate
        # asyncore environments.  Don't bother with the UI.
        state.isTest = True
        testServerReady = threading.Event()
        def runTestServer():
            testSocketMap = {}
            #TestListener(socketMap=testSocketMap)
            testServerReady.set()
            #asyncore.loop(map=testSocketMap)

        proxyReady = threading.Event()
        def runProxy():
            trainer = None
            BayesSMTPProxyListener('localhost', 8025, ('', 8026), trainer)
            proxyReady.set()
            Dibbler.run()

        serverThread = threading.Thread(target=runTestServer)
        serverThread.setDaemon(True)
        serverThread.start()
        testServerReady.wait()
        proxyThread = threading.Thread(target=runProxy)
        proxyThread.setDaemon(True)
        proxyThread.start()
        proxyReady.wait()

    def tearDown(self):
        return
        # Kill the proxy and the test server.
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.connect(('localhost', 8025))
        s.send("kill\r\n")

    def test_direct_connection(self):
        # Connect to the test server.
        smtpServer = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        smtpServer.connect(('localhost', 8025))
        try:
            response = smtpServer.recv(100)
        except socket.error, e:
            if e[0] == 10035:
                # non-blocking socket so that the recognition
                # can proceed, so this doesn't mean much
                pass
            else:
                raise
        self.assertEqual(response, "220 SpamBayes test SMTP server ready\r\n",
                         "Couldn't connect to test SMTP server")
        smtpServer.send('quit\r\n')

    def test_proxy_connection(self):
        # Connect to the proxy server.
        proxy = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        proxy.connect(('localhost', 8026))
        try:
            response = proxy.recv(100)
        except socket.error, e:
            if e[0] == 10035:
                # non-blocking socket so that the recognition
                # can proceed, so this doesn't mean much
                pass
            else:
                raise
        self.assertEqual(response, "220 SpamBayes test SMTP server ready\r\n",
                         "Couldn't connect to proxy server")
        proxy.send('quit\r\n')

    def qtest_disconnection(self):
        proxy = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        proxy.connect(('localhost', 8025))
        try:
            response = proxy.recv(100)
        except socket.error, e:
            if e[0] == 10035:
                # non-blocking socket so that the recognition
                # can proceed, so this doesn't mean much
                pass
            else:
                raise
        proxy.send("quit\r\n")
        try:
            response = proxy.recv(100)
        except socket.error, e:
            if e[0] == 10035:
                # non-blocking socket so that the recognition
                # can proceed, so this doesn't mean much
                pass
            else:
                raise
        self.assertEqual(response, "221 Goodbye\r\n",
                         "Couldn't disconnect from SMTP server")

    def test_sendmessage(self):
        try:
            s = smtplib.SMTP('localhost', 8026)
            s.sendmail("ta-meyer@ihug.co.nz", "ta-meyer@ihug.co.nz", good1)
            s.quit()
        except:
            self.fail("Couldn't send a message through.")

def suite():
    suite = unittest.TestSuite()
    suite.addTest(unittest.makeSuite(SMTPProxyTest))
    return suite

def run():
    # Read the arguments.
    try:
        opts, args = getopt.getopt(sys.argv[1:], 'htz')
    except getopt.error, msg:
        print >>sys.stderr, str(msg) + '\n\n' + __doc__
        sys.exit()

    runSelfTest = False
    for opt, arg in opts:
        if opt == '-h':
            print >>sys.stderr, __doc__
            sys.exit()
        elif opt == '-t':
            state.isTest = True
            state.runTestServer = True
        elif opt == '-z':
            state.isTest = True
            runSelfTest = True

    state.createWorkers()

    if state.runTestServer:
        print "Running a test SMTP server on port 8025..."
        TestListener()
        asyncore.loop()
    else:
        state.buildServerStrings()
        unittest.main(argv=sys.argv + ['suite'])

if __name__ == '__main__':
    run()



From ta-meyer at ihug.co.nz  Wed Sep 24 02:59:48 2003
From: ta-meyer at ihug.co.nz (Tony Meyer)
Date: Wed Sep 24 02:59:57 2003
Subject: [Spambayes-checkins] spambayes/spambayes/test test_smtpproxy.py,
	NONE, 1.1
In-Reply-To: <1ED4ECF91CDED24C8D012BCF2B034F130364A5BD@its-xchg4.massey.ac.nz>
Message-ID: <1ED4ECF91CDED24C8D012BCF2B034F130212AF40@its-xchg4.massey.ac.nz>

> Log Message:
> If the -n switch was before the -d/-p switch, then the name 
> wouldn't be used. This is rather unintuitive, so fix this so 
> that the -d/-p name is used wherever the -n switch is.
> 
> --- NEW FILE: test_smtpproxy.py ---

Drat - I hit "abort" just too late.  I hadn't meant to add this just yet,
since it doesn't work yet.  Oh well; if anyone wants to play about fixing it
up, they're welcome to!  Otherwise it can just sit there not working until I
get a chance to fix it in the weekend.

=Tony Meyer


From mhammond at users.sourceforge.net  Wed Sep 24 20:10:34 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Wed Sep 24 20:10:46 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py,1.14,1.15
Message-ID: 

Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv3347/windows

Modified Files:
	pop3proxy_tray.py 
Log Message:
Patch [ 809008 ] safe start/stop and exlusive execution on windows
Use a mutex to prevent multiple execution of the server on Windows.
pop3proxy tray takes advantage of this, and is also far more resilient
in the face of the world changing underneath it.


Index: pop3proxy_tray.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_tray.py,v
retrieving revision 1.14
retrieving revision 1.15
diff -C2 -d -r1.14 -r1.15
*** pop3proxy_tray.py	24 Sep 2003 01:30:55 -0000	1.14
--- pop3proxy_tray.py	25 Sep 2003 00:10:32 -0000	1.15
***************
*** 26,31 ****
  import thread
  import traceback
- import socket
  
  # This should just be imported from dialogs.dlgutils, but
  # I'm not sure that we can import from the Outlook2000
--- 26,31 ----
  import thread
  import traceback
  
+ verbose = 0
  # This should just be imported from dialogs.dlgutils, but
  # I'm not sure that we can import from the Outlook2000
***************
*** 41,44 ****
--- 41,45 ----
  
  import win32con
+ import winerror
  from win32api import *
  from win32gui import *
***************
*** 108,111 ****
--- 109,127 ----
  serviceName = "pop3proxy"
  
+ def IsServerRunningAnywhere():
+     import win32event
+     mutex_name = "SpamBayesServer"
+     try:
+         hmutex = win32event.CreateMutex(None, True, mutex_name)
+         try:
+             return GetLastError()==winerror.ERROR_ALREADY_EXISTS
+         finally:
+             hmutex.Close()
+     except win32event.error, details:
+         if details[0] != winerror.ERROR_ACCESS_DENIED:
+             raise
+         # Mutex created by some other user - it does exist!
+         return True
+ 
  class MainWindow(object):
      def __init__(self):
***************
*** 114,118 ****
          # that the order is controlled by the id.  Any items where the
          # function is None will appear as separators.
!         self.control_functions = {START_STOP_ID : ("Stop SpamBayes", self.StartStop),
                                    1025 : ("-", None),
                                    1026 : ("Review messages ...", self.OpenReview),
--- 130,134 ----
          # that the order is controlled by the id.  Any items where the
          # function is None will appear as separators.
!         self.control_functions = {START_STOP_ID : ("Stop SpamBayes", self.Stop),
                                    1025 : ("-", None),
                                    1026 : ("Review messages ...", self.OpenReview),
***************
*** 128,133 ****
              WM_TASKBAR_NOTIFY : self.OnTaskbarNotify,
          }
!         self.use_service = True
!         #self.use_service = False
  
          # Create the Window.
--- 144,152 ----
              WM_TASKBAR_NOTIFY : self.OnTaskbarNotify,
          }
!         self.have_prepared_state = False
!         self.last_started_state = None
!         # Only bothering to try the service on Windows NT platforms
!         self.use_service = \
!                 GetVersionEx()[3]==win32con.VER_PLATFORM_WIN32_NT
  
          # Create the Window.
***************
*** 171,211 ****
              "SpamBayes")
          Shell_NotifyIcon(NIM_ADD, nid)
!         self.started = self.IsPortBound()
          self.tip = None
!       
!         try:
!             if self.use_service and self.IsServiceAvailable():
!                 if self.GetServiceStatus() in runningStatus:
!                     self.started = True
!             else:
!                 print "Service not availible. Using thread."
!                 self.use_service = False
!                 self.started = False
!         except:
!             print "Usage of service failed. Reverting to thread."
              self.use_service = False
!             self.started = False
! 
          # Start up sb_server
-         # XXX This needs to be finished off.
-         # XXX This should determine if we are using the service, and if so
-         # XXX start that, and if not kick sb_server off in a separate thread.
-         if not self.use_service:
-             sb_server.prepare(state=sb_server.state)
          if not self.started:
!             self.StartStop()
! 
!     def IsPortBound(self):        
!         s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
!         s.settimeout(0.25)
!         inuse = False
!         try:
!             s.bind(("0.0.0.0",options["html_ui", "port"],))
!         except:
!             inuse = True
!         s.close()
!         s = None
!         return inuse
! 
  
      def BuildToolTip(self):
--- 190,205 ----
              "SpamBayes")
          Shell_NotifyIcon(NIM_ADD, nid)
!         self.started = IsServerRunningAnywhere()
          self.tip = None
!         if self.use_service and not self.IsServiceAvailable():
!             print "Service not availible. Using thread."
              self.use_service = False
!       
          # Start up sb_server
          if not self.started:
!             self.Start()
!         else:
!             print "The server is already running externally - not starting " \
!                   "a local server"
  
      def BuildToolTip(self):
***************
*** 234,250 ****
          if self.started:
              self.control_functions[START_STOP_ID] = ("Stop SpamBayes",
!                                                      self.StartStop)
          else:
              self.control_functions[START_STOP_ID] = ("Start SpamBayes",
!                                          self.StartStop)
          Shell_NotifyIcon(NIM_MODIFY, nid)
  
      def IsServiceAvailable(self):
!         schSCManager = OpenSCManager(None, None, SC_MANAGER_CONNECT)
!         schService   = OpenService(schSCManager, serviceName,
!                                    SERVICE_QUERY_STATUS)
!         if schService:
!             CloseServiceHandle(schService)
!         return schService != None
  
      def GetServiceStatus(self):
--- 228,250 ----
          if self.started:
              self.control_functions[START_STOP_ID] = ("Stop SpamBayes",
!                                                      self.Stop)
          else:
              self.control_functions[START_STOP_ID] = ("Start SpamBayes",
!                                          self.Start)
          Shell_NotifyIcon(NIM_MODIFY, nid)
  
      def IsServiceAvailable(self):
!         try:
!             schSCManager = OpenSCManager(None, None, SC_MANAGER_CONNECT)
!             schService   = OpenService(schSCManager, serviceName,
!                                        SERVICE_QUERY_STATUS)
!             if schService:
!                 CloseServiceHandle(schService)
!             return schService != None
!         except win32api_error, details:
!             if details[0] != winerror.ERROR_SERVICE_DOES_NOT_EXIST:
!                 print "Unexpected windows error querying for service"
!                 print details
!             return False
  
      def GetServiceStatus(self):
***************
*** 340,346 ****
      def OnTaskbarNotify(self, hwnd, msg, wparam, lparam):
          if lparam==win32con.WM_MOUSEMOVE:
!             if self.tip != self.BuildToolTip() or self.started != self.IsPortBound():
!                 self.started = self.IsPortBound()
                  self.UpdateIcon()
          if lparam==win32con.WM_LBUTTONUP:
              # We ignore left clicks
--- 340,347 ----
      def OnTaskbarNotify(self, hwnd, msg, wparam, lparam):
          if lparam==win32con.WM_MOUSEMOVE:
!             if self.tip != self.BuildToolTip():
                  self.UpdateIcon()
+             else:
+                 self.CheckCurrentState()
          if lparam==win32con.WM_LBUTTONUP:
              # We ignore left clicks
***************
*** 353,356 ****
--- 354,360 ----
              self.OpenInterface()
          elif lparam==win32con.WM_RBUTTONUP:
+             # check our state before creating the menu, so it reflects the
+             # true "running state", not just what we thought it was last.
+             self.CheckCurrentState()
              menu = CreatePopupMenu()
              ids = self.control_functions.keys()
***************
*** 391,412 ****
          PostQuitMessage(0)
          
!     def StartProxyThread(self):
!         args = (sb_server.state,)
!         thread.start_new_thread(sb_server.start, args)
          self.started = True
  
!     def StartStop(self):
          if self.use_service:
              if self.GetServiceStatus() in stoppedStatus:
                  self.StartService()
              else:
!                 self.StopService()
          else:
!             if self.started:
                  sb_server.stop(sb_server.state)
!                 self.started = False
              else:
!                 self.StartProxyThread()
!         self.UpdateIcon()
  
      def OpenInterface(self):
--- 395,479 ----
          PostQuitMessage(0)
          
!     def _ProxyThread(self):
          self.started = True
+         try:
+             sb_server.start(sb_server.state)
+         finally:
+             self.started = False
+             self.have_prepared_state = False
  
!     def StartProxyThread(self):
!         thread.start_new_thread(self._ProxyThread, ())
! 
!     def Start(self):
!         self.CheckCurrentState()
!         if self.started:
!             print "Ignoring start request - server already running"
!             return
          if self.use_service:
+             if verbose: print "Doing 'Start' via service"
              if self.GetServiceStatus() in stoppedStatus:
                  self.StartService()
              else:
!                 print "Service was already running - ignoring!"
          else:
!             # Running it internally.
!             if verbose: print "Doing 'Start' internally"
!             if not self.have_prepared_state:
!                 try:
!                     sb_server.prepare(state=sb_server.state)
!                     self.have_prepared_state = True
!                 except sb_server.AlreadyRunningException:
!                     msg = "The proxy is already running on this " \
!                           "machine - please\r\n stop the existing " \
!                           "proxy, and try again."
!                     self.ShowMessage(msg)
!                     return
!             self.StartProxyThread()
!         self.started = True
!         self.UpdateUIState()
!         
!     def Stop(self):
!         self.CheckCurrentState()
!         if not self.started:
!             print "Ignoring stop request - server doesn't appear to be running"
!             return
!         try:
!             use_service = self.use_service
!             if use_service:
!                 # XXX - watch out - if service status is "stopping", trying
!                 # to start is likely to fail until it actually gets to
!                 # "stopped"
!                 if verbose: print "Doing 'Stop' via service"
!                 if self.GetServiceStatus() not in stoppedStatus:
!                     self.StopService()
!                 else:
!                     print "Service was already stopped - weird - falling " \
!                           "back to a socket based quit"
!                     use_service = False
!             if not use_service:
!                 if verbose: print "Stopping local server"
                  sb_server.stop(sb_server.state)
!         except:
!             print "There was an error stopping the server"
!             traceback.print_exc()
!         # but either way, assume it stopped for the sake of our UI
!         self.started = False
!         self.UpdateUIState()
! 
!     def CheckCurrentState(self):
!         self.started = IsServerRunningAnywhere()
!         self.UpdateUIState()
!         
!     def UpdateUIState(self):
!         if self.started != self.last_started_state:
!             self.UpdateIcon()
!             if self.started:
!                 self.control_functions[START_STOP_ID] = ("Stop SpamBayes",
!                                                          self.Stop)
              else:
!                 self.control_functions[START_STOP_ID] = ("Start SpamBayes",
!                                              self.Start)
!             self.last_started_state = self.started
  
      def OpenInterface(self):



From mhammond at users.sourceforge.net  Wed Sep 24 20:10:34 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Wed Sep 24 20:10:50 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_server.py,1.8,1.9
Message-ID: 

Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv3347/scripts

Modified Files:
	sb_server.py 
Log Message:
Patch [ 809008 ] safe start/stop and exlusive execution on windows
Use a mutex to prevent multiple execution of the server on Windows.
pop3proxy tray takes advantage of this, and is also far more resilient
in the face of the world changing underneath it.


Index: sb_server.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_server.py,v
retrieving revision 1.8
retrieving revision 1.9
diff -C2 -d -r1.8 -r1.9
*** sb_server.py	24 Sep 2003 05:28:53 -0000	1.8
--- sb_server.py	25 Sep 2003 00:10:31 -0000	1.9
***************
*** 119,122 ****
--- 119,126 ----
          resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard))
  
+ # exception may be raised if we are already running and check such things.
+ class AlreadyRunningException(Exception):
+     pass
+ 
  # number to add to STAT length for each msg to fudge for spambayes headers
  HEADER_SIZE_FUDGE_FACTOR = 512
***************
*** 548,551 ****
--- 552,598 ----
          return response
  
+ # Implementations of a mutex or other resource which can prevent
+ # multiple servers starting at once.  Platform specific as no reasonable
+ # cross-platform solution exists (however, an old trick is to use a
+ # directory for a mutex, as a "create/test" atomic API generally exists.
+ # Will return a handle to be later closed, or may throw AlreadyRunningException
+ def open_platform_mutex():
+     if sys.platform.startswith("win"):
+         try:
+             import win32event, win32api, winerror, win32con
+             import pywintypes, ntsecuritycon
+             # ideally, the mutex name could include either the username,
+             # or the munged path to the INI file - this would mean we
+             # would allow multiple starts so long as they weren't for
+             # the same user.  However, as of now, the service version
+             # is likely to start as a different user, so a single mutex
+             # is best for now.
+             # XXX - even if we do get clever with another mutex name, we
+             # should consider still creating a non-exclusive
+             # "SpamBayesServer" mutex, if for no better reason than so
+             # an installer can check if we are running
+             mutex_name = "SpamBayesServer"
+             try:
+                 hmutex = win32event.CreateMutex(None, True, mutex_name)
+             except win32event.error, details:
+                 # If another user has the mutex open, we get an "access denied"
+                 # error - this is still telling us what we need to know.
+                 if details[0] != winerror.ERROR_ACCESS_DENIED:
+                     raise
+                 raise AlreadyRunningException
+             # mutex opened - now check if we actually created it.
+             if win32api.GetLastError()==winerror.ERROR_ALREADY_EXISTS:
+                 win32api.CloseHandle(hmutex)
+                 raise AlreadyRunningException
+             return hmutex
+         except ImportError:
+             # no win32all - no worries, just start
+             pass
+     return None
+ 
+ def close_platform_mutex(mutex):
+     if sys.platform.startswith("win"):
+         if mutex is not None:
+             mutex.Close()
  
  # This keeps the global state of the module - the command-line options,
***************
*** 558,565 ****
          and are then overridden by the command-line processing code in the
          __main__ code below."""
          # Open the log file.
          if options["globals", "verbose"]:
              self.logFile = open('_pop3proxy.log', 'wb', 0)
- 
          self.servers = []
          self.proxyPorts = []
--- 605,619 ----
          and are then overridden by the command-line processing code in the
          __main__ code below."""
+         self.logFile = None
+         self.bayes = None
+         self.platform_mutex = None
+         self.prepared = False
+         self.init()
+ 
+     def init(self):
+         assert not self.prepared, "init after prepare, but before close"
          # Open the log file.
          if options["globals", "verbose"]:
              self.logFile = open('_pop3proxy.log', 'wb', 0)
          self.servers = []
          self.proxyPorts = []
***************
*** 601,604 ****
--- 655,683 ----
          self.uniquifier = 2
  
+     def close(self):
+         assert self.prepared, "closed without being prepared!"
+         self.servers = None
+         if self.bayes is not None:
+             # Only store a non-empty db.
+             if self.bayes.nham != 0 and self.bayes.nspam != 0:
+                 state.bayes.store()
+             self.bayes.close()
+             self.bayes = None
+ 
+         self.spamCorpus = self.hamCorpus = self.unknownCorpus = None
+         self.spamTrainer = self.hamTrainer = None
+ 
+         self.prepared = False
+         close_platform_mutex(self.platform_mutex)
+         self.platform_mutex = None
+ 
+     def prepare(self):
+         # If we can, prevent multiple servers from running at the same time.
+         self.platform_mutex = open_platform_mutex()
+     
+         # Do whatever we've been asked to do...
+         self.createWorkers()
+         self.prepared = True
+         
      def buildServerStrings(self):
          """After the server details have been set up, this creates string
***************
*** 717,732 ****
      # affect any running proxies - once a listener has created a proxy,
      # that proxy is then independent of it.
      for proxy in proxyListeners:
          proxy.close()
      del proxyListeners[:]
  
!     # Close the database (if there is one); we should anyway, and gdbm
!     # complains if we try to reopen it without closing it first.
!     if hasattr(state, "bayes"):
!         # Only store a non-empty db.
!         if state.bayes.nham != 0 and state.bayes.nspam != 0:
!             state.bayes.store()
!         state.bayes.close()
! 
      state = State()
  
--- 796,807 ----
      # affect any running proxies - once a listener has created a proxy,
      # that proxy is then independent of it.
+     # (but won't closing the database screw them?)
      for proxy in proxyListeners:
          proxy.close()
      del proxyListeners[:]
  
!     # Close the state (which saves if necessary)
!     state.close()
!     # And get a new one going.
      state = State()
  
***************
*** 746,752 ****
  
  def prepare(state):
!     # Do whatever we've been asked to do...
!     state.createWorkers()
! 
      # Launch any SMTP proxies.  Note that if the user hasn't specified any
      # SMTP proxy information in their configuration, then nothing will
--- 821,826 ----
  
  def prepare(state):
!     state.init()
!     state.prepare()
      # Launch any SMTP proxies.  Note that if the user hasn't specified any
      # SMTP proxy information in their configuration, then nothing will
***************
*** 761,766 ****
  
  def start(state):
!     # kick everything off    
!     main(state.servers, state.proxyPorts, state.uiPort, state.launchUI)
  
  def stop(state):
--- 835,844 ----
  
  def start(state):
!     # kick everything off
!     assert state.prepared, "starting before preparing state"
!     try:
!         main(state.servers, state.proxyPorts, state.uiPort, state.launchUI)
!     finally:
!         state.close()
  
  def stop(state):
***************
*** 821,825 ****
              state.proxyPorts = [('', 110)]
  
!         prepare(state=state)
          start(state=state)
  
--- 899,909 ----
              state.proxyPorts = [('', 110)]
  
!         try:
!             prepare(state=state)
!         except AlreadyRunningException:
!             print  >>sys.stderr, \
!                    "ERROR: The proxy is already running on this machine."
!             print  >>sys.stderr, "Please stop the existing proxy and try again"
!             return
          start(state=state)
  



From anadelonbrin at users.sourceforge.net  Wed Sep 24 21:02:52 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Wed Sep 24 21:02:55 2003
Subject: [Spambayes-checkins] spambayes README.txt,1.53,1.54
Message-ID: 

Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv11447

Modified Files:
	README.txt 
Log Message:
Point out that if you want to use the service, you need to install the
win32 extensions.

Fix an error in the imapfilter instructions, as it forces you to use one of the switches.

Index: README.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/README.txt,v
retrieving revision 1.53
retrieving revision 1.54
diff -C2 -d -r1.53 -r1.54
*** README.txt	9 Sep 2003 18:30:34 -0000	1.53
--- README.txt	25 Sep 2003 01:02:50 -0000	1.54
***************
*** 113,117 ****
  Now launch pop3proxy, either by running the "pop3proxy_service.py" script
  (for those using Windows 2000, Windows NT or Windows XP), or the
! "sb_server.py" script (for everyone else).
  
  All you need to do to configure SpamBayes is to open a web page to
--- 113,121 ----
  Now launch pop3proxy, either by running the "pop3proxy_service.py" script
  (for those using Windows 2000, Windows NT or Windows XP), or the
! "sb_server.py" script (for everyone else).  Note that if you want to use
! the service, you need to also have Mark Hammond's win32 extensions for
! Python installed:
! 
! 
  
  All you need to do to configure SpamBayes is to open a web page to
***************
*** 138,142 ****
  -----------
  
! To configure SpamBayes, run the "sb_imapfilter.py" script, and open a web
  page to , click on the "Configuration" link at the
  top right, and fill in the relevant details.  Everything should be ok with
--- 142,146 ----
  -----------
  
! To configure SpamBayes, run "sb_imapfilter.py -b", which should open a web
  page to , click on the "Configuration" link at the
  top right, and fill in the relevant details.  Everything should be ok with



From anadelonbrin at users.sourceforge.net  Wed Sep 24 21:03:50 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Wed Sep 24 21:03:53 2003
Subject: [Spambayes-checkins] spambayes/utilities which_database.py,1.2,1.3
Message-ID: 

Update of /cvsroot/spambayes/spambayes/utilities
In directory sc8-pr-cvs1:/tmp/cvs-serv11651/utilities

Modified Files:
	which_database.py 
Log Message:
Don't use 'str' as a variable name.

Print out a more informative message than "Your database X is a None" when
checking fails.

Index: which_database.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/utilities/which_database.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** which_database.py	16 Aug 2003 07:46:41 -0000	1.2
--- which_database.py	25 Sep 2003 01:03:48 -0000	1.3
***************
*** 79,84 ****
              return
  
!     str = whichdb.whichdb(hammie)
!     if str == "dbhash":
          # could be dbhash or bsddb3
          try:
--- 79,84 ----
              return
  
!     db_type = whichdb.whichdb(hammie)
!     if db_type == "dbhash":
          # could be dbhash or bsddb3
          try:
***************
*** 87,91 ****
              print "Your storage %s is a: bsddb3" % (hammie,)
              return
!     print "Your storage %s is a: %s" % (hammie, str)
  
  if __name__ == "__main__":
--- 87,94 ----
              print "Your storage %s is a: bsddb3" % (hammie,)
              return
!     elif db_type is None:
!         print "Your storage %s either does not exist, or is unreadable." % \
!               (hammie,)
!     print "Your storage %s is a: %s" % (hammie, db_type)
  
  if __name__ == "__main__":



From anadelonbrin at users.sourceforge.net  Wed Sep 24 21:01:52 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Wed Sep 24 21:06:17 2003
Subject: [Spambayes-checkins] spambayes README.txt,1.53,1.53.2.1
Message-ID: 

Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv11112

Modified Files:
      Tag: release_1_0
	README.txt 
Log Message:
Point out that if you want to use the service, you need to install the
win32 extensions.

Index: README.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/README.txt,v
retrieving revision 1.53
retrieving revision 1.53.2.1
diff -C2 -d -r1.53 -r1.53.2.1
*** README.txt	9 Sep 2003 18:30:34 -0000	1.53
--- README.txt	25 Sep 2003 01:01:49 -0000	1.53.2.1
***************
*** 113,117 ****
  Now launch pop3proxy, either by running the "pop3proxy_service.py" script
  (for those using Windows 2000, Windows NT or Windows XP), or the
! "sb_server.py" script (for everyone else).
  
  All you need to do to configure SpamBayes is to open a web page to
--- 113,121 ----
  Now launch pop3proxy, either by running the "pop3proxy_service.py" script
  (for those using Windows 2000, Windows NT or Windows XP), or the
! "sb_server.py" script (for everyone else).  Note that if you want to use
! the service, you need to also have Mark Hammond's win32 extensions for
! Python installed:
! 
! 
  
  All you need to do to configure SpamBayes is to open a web page to



From anadelonbrin at users.sourceforge.net  Wed Sep 24 21:20:22 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Wed Sep 24 21:20:36 2003
Subject: [Spambayes-checkins] website docs.ht,1.16,1.17 unix.ht,1.6,1.7
Message-ID: 

Update of /cvsroot/spambayes/website
In directory sc8-pr-cvs1:/tmp/cvs-serv16193

Modified Files:
	docs.ht unix.ht 
Log Message:
Update the docs to reflect current reality.

Index: docs.ht
===================================================================
RCS file: /cvsroot/spambayes/website/docs.ht,v
retrieving revision 1.16
retrieving revision 1.17
diff -C2 -d -r1.16 -r1.17
*** docs.ht	15 Sep 2003 00:37:50 -0000	1.16
--- docs.ht	25 Sep 2003 01:20:20 -0000	1.17
***************
*** 5,18 ****
  
                Project documentation
                
  
                  
- 
                • The README.txt file from the project CVS -- a collection of miscellaneous clues of what's what in the project.
- 
                • The TESTING.txt file -- Clues about the practice of statistical testing, adapted from Tim
-  comments on python-dev.
  
                • A rudimentary set of Frequently Asked Questions (FAQ).
                  • 
! 
                • Instructions on integrating Spambayes into your mail system.
                  • 
  
                • The Outlook plugin includes an "About" File, and a 
  "Troubleshooting Guide" that can be accessed via the toolbar.
  (Note that the online documentaton is always for the latest source version, and so might not correspond exactly with the version you are using.
  Always start with the documentation that came with the version you installed.)
                  • 
! 
                • There's also a vast number of clues and notes scattered as block comments through the code.
  
                
  
--- 5,18 ----
  
                Project documentation
                
  
                  
  
                • A rudimentary set of Frequently Asked Questions (FAQ).
                  • 
! 
                • Instructions on installing Spambayes and integrating it into your mail system.
                  • 
  
                • The Outlook plugin includes an "About" File, and a 
  "Troubleshooting Guide" that can be accessed via the toolbar.
  (Note that the online documentaton is always for the latest source version, and so might not correspond exactly with the version you are using.
  Always start with the documentation that came with the version you installed.)
                  • 
! 
                • The README-DEVEL.txt information that should be of use to people planning on developing code based on SpamBayes.
                  • 
! 
                • The TESTING.txt file -- Clues about the practice of statistical testing, adapted from Tim
!  comments on python-dev.
! 
                • There are also a vast number of clues and notes scattered as block comments through the code.
  
                
  

Index: unix.ht
===================================================================
RCS file: /cvsroot/spambayes/website/unix.ht,v
retrieving revision 1.6
retrieving revision 1.7
diff -C2 -d -r1.6 -r1.7
*** unix.ht	13 Aug 2003 15:00:54 -0000	1.6
--- unix.ht	25 Sep 2003 01:20:20 -0000	1.7
***************
*** 48,53 ****
  
                
  
                Additional details are available in the Hammie
! integration guide.
                
  
  
                POP3
                
--- 48,53 ----
  
                
  
                Additional details are available in the Hammie
! readme.
                
  
  
                POP3
                
***************
*** 64,69 ****
  
  
                See the Hammie
! integration guide for a detailed discussion of the many training options
  on Unix systems.
  
--- 64,69 ----
  
  
                See the Hammie
! readme for a detailed discussion of the many training options
  on Unix systems.
  



From montanaro at users.sourceforge.net  Thu Sep 25 09:13:11 2003
From: montanaro at users.sourceforge.net (Skip Montanaro)
Date: Thu Sep 25 09:13:20 2003
Subject: [Spambayes-checkins] website faq.txt,1.44,1.45
Message-ID: 

Update of /cvsroot/spambayes/website
In directory sc8-pr-cvs1:/tmp/cvs-serv2639

Modified Files:
	faq.txt 
Log Message:
Mention OL 2003 beta, which seems to cause people occasional problems.


Index: faq.txt
===================================================================
RCS file: /cvsroot/spambayes/website/faq.txt,v
retrieving revision 1.44
retrieving revision 1.45
diff -C2 -d -r1.44 -r1.45
*** faq.txt	11 Sep 2003 07:58:34 -0000	1.44
--- faq.txt	25 Sep 2003 13:13:08 -0000	1.45
***************
*** 432,435 ****
--- 432,439 ----
  know if this is not the case.  The `troubleshooting guide`_ for the Outlook
  plugin contains the most up-to-date help for working around known problems.
+ A number of people have used the plugin with a beta version of Outlook 2003.
+ If you fall into that category, note that you must have applied all the
+ technical refreshes released by Microsoft to use the plugin successfully.
+ Better yet, upgrade to the final version when it's available.
  
  .. _troubleshooting guide: http://cvs.sourceforge.net/cgi-bin/viewcvs.cgi/*checkout*/spambayes/spambayes/Outlook2000/docs/troubleshooting.html?rev=HEAD&content-type=text/plain



From montanaro at users.sourceforge.net  Thu Sep 25 10:22:32 2003
From: montanaro at users.sourceforge.net (Skip Montanaro)
Date: Thu Sep 25 10:22:40 2003
Subject: [Spambayes-checkins] spambayes/contrib nway.py,1.4,1.5
Message-ID: 

Update of /cvsroot/spambayes/spambayes/contrib
In directory sc8-pr-cvs1:/tmp/cvs-serv17006

Modified Files:
	nway.py 
Log Message:
elaborate usage


Index: nway.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/contrib/nway.py,v
retrieving revision 1.4
retrieving revision 1.5
diff -C2 -d -r1.4 -r1.5
*** nway.py	10 Sep 2003 04:33:17 -0000	1.4
--- nway.py	25 Sep 2003 14:22:30 -0000	1.5
***************
*** 18,26 ****
  
  Training is left up to the user.  In general, you want to train so that a
! message in a particular category will score as spam when checked against
! that category's training database.  For example, suppose you have the
! following mbox formatted files: python, music, family, cars.  If you wanted
! to create a training database for each of them you could execute this
! series of mboxtrain.py commands:
  
      sb_mboxtrain.py -f -d python.db -s python -g music -g family -g cars
--- 18,26 ----
  
  Training is left up to the user.  In general, you want to train so that a
! message in a particular category will score above your spam threshold when
! checked against that category's training database.  For example, suppose you
! have the following mbox formatted files: python, music, family, cars.  If
! you wanted to create a training database for each of them you could execute
! this series of mboxtrain.py commands:
  
      sb_mboxtrain.py -f -d python.db -s python -g music -g family -g cars
***************
*** 32,35 ****
--- 32,65 ----
  
      %(prog)s python=python.db music=music.db family=family.db cars=cars.db
+ 
+ Normal usage (at least as I envisioned it) would be to run the program via
+ procmail or something similar.  You'd then have a .procmailrc file which
+ looked something like this:
+ 
+     :0 fw:sb.lock
+     | $(prog)s spam=spam.db python=python.db music=music.db ...
+ 
+     :0
+     * ^X-Spambayes-Classification: spam
+     spam
+ 
+     :0
+     * ^X-Spambayes-Classification: python
+     python
+ 
+     :0
+     * ^X-Spambayes-Classification: music
+     music
+ 
+     ...
+ 
+     :0
+     * ^X-Spambayes-Classification: unsure
+     unsure
+ 
+ Note that I've not tried this (yet).  It should simplify the logic in a
+ .procmailrc file and probably classify messages better than writing more
+ convoluted procmail rules.
+ 
  """
  



From anadelonbrin at users.sourceforge.net  Thu Sep 25 23:46:40 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 25 23:46:47 2003
Subject: [Spambayes-checkins] spambayes README.txt,1.54,1.55
Message-ID: 

Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv6296

Modified Files:
	README.txt 
Log Message:
Clarify the 'really impatient' instructions: installing is still necessary (or setting the pythonpath)
and the sb_server.py file is in the scripts directory.

Index: README.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/README.txt,v
retrieving revision 1.54
retrieving revision 1.55
diff -C2 -d -r1.54 -r1.55
*** README.txt	25 Sep 2003 01:02:50 -0000	1.54
--- README.txt	26 Sep 2003 03:46:38 -0000	1.55
***************
*** 67,74 ****
  If you get your mail from a POP3 server, then all you should need to do
  to get running is change your mail client to send and receive mail from
! "localhost", and then run "python sb_server.py -b" in the directory you
! expanded the SpamBayes source into.  This will open a web browser window
! - click the "Configuration" link at the top right and fill in the various
! settings.
  
  
--- 67,74 ----
  If you get your mail from a POP3 server, then all you should need to do
  to get running is change your mail client to send and receive mail from
! "localhost", and then run "python setup.py install" and then
! "python scripts/sb_server.py -b" in the directory you expanded the
! SpamBayes source into.  This will open a web browser window - click the
! "Configuration" link at the top right and fill in the various settings.
  
  



From anadelonbrin at users.sourceforge.net  Thu Sep 25 23:52:00 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 25 23:52:03 2003
Subject: [Spambayes-checkins] spambayes README.txt,1.53.2.1,1.53.2.2
Message-ID: 

Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv7036

Modified Files:
      Tag: release_1_0
	README.txt 
Log Message:
Clarify the 'really impatient' instructions: installing is still necessary (or setting the pythonpath)
and the sb_server.py file is in the scripts directory.

Index: README.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/README.txt,v
retrieving revision 1.53.2.1
retrieving revision 1.53.2.2
diff -C2 -d -r1.53.2.1 -r1.53.2.2
*** README.txt	25 Sep 2003 01:01:49 -0000	1.53.2.1
--- README.txt	26 Sep 2003 03:51:57 -0000	1.53.2.2
***************
*** 67,74 ****
  If you get your mail from a POP3 server, then all you should need to do
  to get running is change your mail client to send and receive mail from
! "localhost", and then run "python sb_server.py -b" in the directory you
! expanded the SpamBayes source into.  This will open a web browser window
! - click the "Configuration" link at the top right and fill in the various
! settings.
  
  
--- 67,74 ----
  If you get your mail from a POP3 server, then all you should need to do
  to get running is change your mail client to send and receive mail from
! "localhost", and then run "python setup.py install" and then
! "python scripts/sb_server.py -b" in the directory you expanded the
! SpamBayes source into.  This will open a web browser window - click the
! "Configuration" link at the top right and fill in the various settings.
  
  
***************
*** 111,119 ****
  "smtp.example.com".
  
! Now launch pop3proxy, either by running the "pop3proxy_service.py" script
! (for those using Windows 2000, Windows NT or Windows XP), or the
! "sb_server.py" script (for everyone else).  Note that if you want to use
! the service, you need to also have Mark Hammond's win32 extensions for
! Python installed:
  
  
--- 111,119 ----
  "smtp.example.com".
  
! Now launch SpamBayes, either by running "pop3proxy_service.py install"
! and then "net start pop3proxy" (for those using Windows 2000, Windows NT
! or Windows XP), or the "sb_server.py" script (for everyone else).  Note
! that if you want to use the service, you need to also have Mark
! Hammond's win32 extensions for Python installed:
  
  



From anadelonbrin at users.sourceforge.net  Thu Sep 25 23:54:40 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 25 23:54:43 2003
Subject: [Spambayes-checkins] spambayes README.txt,1.55,1.56
Message-ID: 

Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv7451

Modified Files:
	README.txt 
Log Message:
Instructions for using the service weren't correct - you need to run with "install",
and then start it with "net start pop3proxy", not just run the script.

Index: README.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/README.txt,v
retrieving revision 1.55
retrieving revision 1.56
diff -C2 -d -r1.55 -r1.56
*** README.txt	26 Sep 2003 03:46:38 -0000	1.55
--- README.txt	26 Sep 2003 03:54:38 -0000	1.56
***************
*** 111,119 ****
  "smtp.example.com".
  
! Now launch pop3proxy, either by running the "pop3proxy_service.py" script
! (for those using Windows 2000, Windows NT or Windows XP), or the
! "sb_server.py" script (for everyone else).  Note that if you want to use
! the service, you need to also have Mark Hammond's win32 extensions for
! Python installed:
  
  
--- 111,119 ----
  "smtp.example.com".
  
! Now launch SpamBayes, either by running "pop3proxy_service.py install"
! and then "net start pop3proxy" (for those using Windows 2000, Windows NT
! or Windows XP), or the "sb_server.py" script (for everyone else).  Note
! that if you want to use the service, you need to also have Mark
! Hammond's win32 extensions for Python installed:
  
  



From anadelonbrin at users.sourceforge.net  Fri Sep 26 00:23:51 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 26 00:23:54 2003
Subject: [Spambayes-checkins] spambayes/utilities which_database.py,1.3,1.4
Message-ID: 

Update of /cvsroot/spambayes/spambayes/utilities
In directory sc8-pr-cvs1:/tmp/cvs-serv11439/utilities

Modified Files:
	which_database.py 
Log Message:
Distinguish between a database not existing yet and not being able to be read.


Index: which_database.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/utilities/which_database.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** which_database.py	25 Sep 2003 01:03:48 -0000	1.3
--- which_database.py	26 Sep 2003 04:23:49 -0000	1.4
***************
*** 79,82 ****
--- 79,85 ----
              return
  
+     if not os.path.exists(hammie):
+         print "Your storage file does not exist yet."
+         return
      db_type = whichdb.whichdb(hammie)
      if db_type == "dbhash":
***************
*** 88,93 ****
              return
      elif db_type is None:
!         print "Your storage %s either does not exist, or is unreadable." % \
!               (hammie,)
      print "Your storage %s is a: %s" % (hammie, db_type)
  
--- 91,95 ----
              return
      elif db_type is None:
!         print "Your storage %s is unreadable." % (hammie,)
      print "Your storage %s is a: %s" % (hammie, db_type)
  



From anadelonbrin at users.sourceforge.net  Fri Sep 26 02:12:15 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 26 02:12:19 2003
Subject: [Spambayes-checkins] spambayes setup.py,1.26,1.27
Message-ID: 

Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv26315

Modified Files:
	setup.py 
Log Message:
If we are using windows, we really should install the pop3proxy_service and
pop3proxy_tray scripts as well, as they may be needed, and the rest of the
expanded archive isn't, once we have installed.

Index: setup.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/setup.py,v
retrieving revision 1.26
retrieving revision 1.27
diff -C2 -d -r1.26 -r1.27
*** setup.py	21 Sep 2003 04:30:08 -0000	1.26
--- setup.py	26 Sep 2003 06:12:13 -0000	1.27
***************
*** 74,77 ****
--- 74,100 ----
          return parent.run(self)
  
+ scripts=['scripts/sb_client.py',
+          'scripts/sb_dbexpimp.py',
+          'scripts/sb_filter.py',
+          'scripts/sb_imapfilter.py',
+          'scripts/sb_mailsort.py',
+          'scripts/sb_mboxtrain.py',
+          'scripts/sb_notesfilter.py',
+          'scripts/sb_pop3dnd.py',
+          'scripts/sb_server.py',
+          'scripts/sb_unheader.py',
+          'scripts/sb_upload.py',
+          'scripts/sb_xmlrpcserver.py',
+          'scripts/sb_chkopts.py',
+         ]
+ 
+ if sys.platform == 'win32':
+     # Also install the pop3proxy_service and pop3proxy_tray scripts.
+     # pop3proxy_service is only needed for installation and removal,
+     # but pop3proxy_tray needs to be used all the time.  Neither is
+     # any use on a non-win32 platform.
+     scripts.append('windows/pop3proxy_service.py')
+     scripts.append('windows/pop3proxy_tray.py')
+ 
  setup(
      name='spambayes',
***************
*** 82,99 ****
      url = "http://spambayes.sourceforge.net",
      cmdclass = {'install_scripts': install_scripts},
!     scripts=['scripts/sb_client.py',
!              'scripts/sb_dbexpimp.py',
!              'scripts/sb_filter.py',
!              'scripts/sb_imapfilter.py',
!              'scripts/sb_mailsort.py',
!              'scripts/sb_mboxtrain.py',
!              'scripts/sb_notesfilter.py',
!              'scripts/sb_pop3dnd.py',
!              'scripts/sb_server.py',
!              'scripts/sb_unheader.py',
!              'scripts/sb_upload.py',
!              'scripts/sb_xmlrpcserver.py',
!              'scripts/sb_chkopts.py',
!             ],
      packages = [
          'spambayes',
--- 105,109 ----
      url = "http://spambayes.sourceforge.net",
      cmdclass = {'install_scripts': install_scripts},
!     scripts=scripts,
      packages = [
          'spambayes',



From anadelonbrin at users.sourceforge.net  Fri Sep 26 02:12:38 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 26 02:12:41 2003
Subject: [Spambayes-checkins] spambayes setup.py,1.24.2.1,1.24.2.2
Message-ID: 

Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv26372

Modified Files:
      Tag: release_1_0
	setup.py 
Log Message:
If we are using windows, we really should install the pop3proxy_service and
pop3proxy_tray scripts as well, as they may be needed, and the rest of the
expanded archive isn't, once we have installed.

Index: setup.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/setup.py,v
retrieving revision 1.24.2.1
retrieving revision 1.24.2.2
diff -C2 -d -r1.24.2.1 -r1.24.2.2
*** setup.py	21 Sep 2003 04:29:38 -0000	1.24.2.1
--- setup.py	26 Sep 2003 06:12:36 -0000	1.24.2.2
***************
*** 73,76 ****
--- 73,100 ----
          return parent.run(self)
  
+ scripts=['scripts/sb_client.py',
+          'scripts/sb_dbexpimp.py',
+          'scripts/sb_filter.py',
+          'scripts/sb_imapfilter.py',
+          'scripts/sb_mailsort.py',
+          'scripts/sb_mboxtrain.py',
+          'scripts/sb_notesfilter.py',
+          'scripts/sb_pop3dnd.py',
+          'scripts/sb_server.py',
+          'scripts/sb_smtpproxy.py',
+          'scripts/sb_unheader.py',
+          'scripts/sb_upload.py',
+          'scripts/sb_xmlrpcserver.py',
+          'scripts/sb_chkopts.py',
+         ]
+ 
+ if sys.platform == 'win32':
+     # Also install the pop3proxy_service and pop3proxy_tray scripts.
+     # pop3proxy_service is only needed for installation and removal,
+     # but pop3proxy_tray needs to be used all the time.  Neither is
+     # any use on a non-win32 platform.
+     scripts.append('windows/pop3proxy_service.py')
+     scripts.append('windows/pop3proxy_tray.py')
+ 
  setup(
      name='spambayes',
***************
*** 81,99 ****
      url = "http://spambayes.sourceforge.net",
      cmdclass = {'install_scripts': install_scripts},
!     scripts=['scripts/sb_client.py',
!              'scripts/sb_dbexpimp.py',
!              'scripts/sb_filter.py',
!              'scripts/sb_imapfilter.py',
!              'scripts/sb_mailsort.py',
!              'scripts/sb_mboxtrain.py',
!              'scripts/sb_notesfilter.py',
!              'scripts/sb_pop3dnd.py',
!              'scripts/sb_server.py',
!              'scripts/sb_smtpproxy.py',
!              'scripts/sb_unheader.py',
!              'scripts/sb_upload.py',
!              'scripts/sb_xmlrpcserver.py',
!              'scripts/sb_chkopts.py',
!             ],
      packages = [
          'spambayes',
--- 105,109 ----
      url = "http://spambayes.sourceforge.net",
      cmdclass = {'install_scripts': install_scripts},
!     scripts=scripts,
      packages = [
          'spambayes',



From montanaro at users.sourceforge.net  Fri Sep 26 13:55:38 2003
From: montanaro at users.sourceforge.net (Skip Montanaro)
Date: Fri Sep 26 13:55:41 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_mboxtrain.py,1.3,1.4
Message-ID: 

Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv5228

Modified Files:
	sb_mboxtrain.py 
Log Message:
* reflow a couple long lines
* dump an unused import
* initialize TRAINED_HDR from the options file


Index: sb_mboxtrain.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_mboxtrain.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** sb_mboxtrain.py	18 Sep 2003 15:36:30 -0000	1.3
--- sb_mboxtrain.py	26 Sep 2003 17:55:36 -0000	1.4
***************
*** 34,39 ****
          quiet mode; no output
          
!     -n  train mail residing in "new" directory, in addition to "cur" directory,
!         which is always trained (Maildir only)
  
      -r  remove mail which was trained on (Maildir only)
--- 34,39 ----
          quiet mode; no output
          
!     -n  train mail residing in "new" directory, in addition to "cur"
!         directory, which is always trained (Maildir only)
  
      -r  remove mail which was trained on (Maildir only)
***************
*** 51,55 ****
  
  program = sys.argv[0]
! TRAINED_HDR = "X-Spambayes-Trained"
  loud = True
  
--- 51,55 ----
  
  program = sys.argv[0]
! TRAINED_HDR = options["Headers", "trained_header_name"]
  loud = True
  
***************
*** 139,143 ****
      import mailbox
      import fcntl
-     import tempfile
  
      # Open and lock the mailbox.  Some systems require it be opened for
--- 139,142 ----
***************
*** 230,236 ****
          mbox_train(h, path, is_spam, force)
      elif os.path.isdir(os.path.join(path, "cur")):
!         maildir_train(h, os.path.join(path, "cur"), is_spam, force, removetrained)
          if trainnew:
!             maildir_train(h, os.path.join(path, "new"), is_spam, force, removetrained)
      elif os.path.isdir(path):
          mhdir_train(h, path, is_spam, force)
--- 229,237 ----
          mbox_train(h, path, is_spam, force)
      elif os.path.isdir(os.path.join(path, "cur")):
!         maildir_train(h, os.path.join(path, "cur"), is_spam, force,
!                       removetrained)
          if trainnew:
!             maildir_train(h, os.path.join(path, "new"), is_spam, force,
!                           removetrained)
      elif os.path.isdir(path):
          mhdir_train(h, path, is_spam, force)



From montanaro at users.sourceforge.net  Fri Sep 26 14:16:32 2003
From: montanaro at users.sourceforge.net (Skip Montanaro)
Date: Fri Sep 26 14:16:39 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_mboxtrain.py,1.4,1.5
Message-ID: 

Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv9081

Modified Files:
	sb_mboxtrain.py 
Log Message:
Dump TRAINED_HDR global altogether and just reference options["Headers",
"trained_header_name"] directly.


Index: sb_mboxtrain.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_mboxtrain.py,v
retrieving revision 1.4
retrieving revision 1.5
diff -C2 -d -r1.4 -r1.5
*** sb_mboxtrain.py	26 Sep 2003 17:55:36 -0000	1.4
--- sb_mboxtrain.py	26 Sep 2003 18:16:30 -0000	1.5
***************
*** 51,55 ****
  
  program = sys.argv[0]
- TRAINED_HDR = options["Headers", "trained_header_name"]
  loud = True
  
--- 51,54 ----
***************
*** 69,77 ****
      else:
          spamtxt = options["Headers", "header_ham_string"]
!     oldtxt = msg.get(TRAINED_HDR)
      if force:
          # Train no matter what.
          if oldtxt != None:
!             del msg[TRAINED_HDR]
      elif oldtxt == spamtxt:
          # Skip this one, we've already trained with it.
--- 68,76 ----
      else:
          spamtxt = options["Headers", "header_ham_string"]
!     oldtxt = msg.get(options["Headers", "trained_header_name"])
      if force:
          # Train no matter what.
          if oldtxt != None:
!             del msg[options["Headers", "trained_header_name"]]
      elif oldtxt == spamtxt:
          # Skip this one, we've already trained with it.
***************
*** 79,86 ****
      elif oldtxt != None:
          # It's been trained, but as something else.  Untrain.
!         del msg[TRAINED_HDR]
          h.untrain(msg, not is_spam)
      h.train(msg, is_spam)
!     msg.add_header(TRAINED_HDR, spamtxt)
  
      return True
--- 78,85 ----
      elif oldtxt != None:
          # It's been trained, but as something else.  Untrain.
!         del msg[options["Headers", "trained_header_name"]]
          h.untrain(msg, not is_spam)
      h.train(msg, is_spam)
!     msg.add_header(options["Headers", "trained_header_name"], spamtxt)
  
      return True
***************
*** 178,181 ****
--- 177,181 ----
                                   "may be corrupted.")
              raise
+ 
      fcntl.lockf(f, fcntl.LOCK_UN)
      f.close()



From montanaro at users.sourceforge.net  Fri Sep 26 14:17:46 2003
From: montanaro at users.sourceforge.net (Skip Montanaro)
Date: Fri Sep 26 14:18:45 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_mboxtrain.py,1.5,1.6
Message-ID: 

Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv9277

Modified Files:
	sb_mboxtrain.py 
Log Message:
Sense of test for include_trained option was incorrect in mbox_train.


Index: sb_mboxtrain.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_mboxtrain.py,v
retrieving revision 1.5
retrieving revision 1.6
diff -C2 -d -r1.5 -r1.6
*** sb_mboxtrain.py	26 Sep 2003 18:16:30 -0000	1.5
--- sb_mboxtrain.py	26 Sep 2003 18:17:44 -0000	1.6
***************
*** 156,164 ****
          if msg_train(h, msg, is_spam, force):
              trained += 1
!         if not options["Headers", "include_trained"]:
              # Write it out with the Unix "From " line
              outf.write(msg.as_string(True))
  
!     if not options["Headers", "include_trained"]:
          outf.seek(0)
          try:
--- 156,164 ----
          if msg_train(h, msg, is_spam, force):
              trained += 1
!         if options["Headers", "include_trained"]:
              # Write it out with the Unix "From " line
              outf.write(msg.as_string(True))
  
!     if options["Headers", "include_trained"]:
          outf.seek(0)
          try:



From montanaro at users.sourceforge.net  Fri Sep 26 14:20:05 2003
From: montanaro at users.sourceforge.net (Skip Montanaro)
Date: Fri Sep 26 14:20:29 2003
Subject: [Spambayes-checkins] spambayes CHANGELOG.txt,1.19,1.20
Message-ID: 

Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv9653

Modified Files:
	CHANGELOG.txt 
Log Message:
log recent changes


Index: CHANGELOG.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/CHANGELOG.txt,v
retrieving revision 1.19
retrieving revision 1.20
diff -C2 -d -r1.19 -r1.20
*** CHANGELOG.txt	18 Sep 2003 22:20:36 -0000	1.19
--- CHANGELOG.txt	26 Sep 2003 18:20:03 -0000	1.20
***************
*** 1,4 ****
--- 1,9 ----
  [Note that all dates are in English, not American format - i.e. day/month/year]
  
+ Alpha Release 7
+ ===============
+ Skip Montanaro    26/09/2003  Correct sense of include_trained test in mbox_train.
+ Skip Montanaro    26/09/2003  Dump TRAINED_HDR global.  Reference options[...] instead.
+ 
  Alpha Release 6
  ===============



From anadelonbrin at users.sourceforge.net  Sat Sep 27 22:40:11 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sat Sep 27 22:40:15 2003
Subject: [Spambayes-checkins] spambayes/spambayes/resources ui.html, 1.19,
	1.19.2.1
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes/resources
In directory sc8-pr-cvs1:/tmp/cvs-serv11583/spambayes/resources

Modified Files:
      Tag: release_1_0
	ui.html 
Log Message:
Anthony says pragma: no_cache is a no-no, so away it goes.

Index: ui.html
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/resources/ui.html,v
retrieving revision 1.19
retrieving revision 1.19.2.1
diff -C2 -d -r1.19 -r1.19.2.1
*** ui.html	30 Aug 2003 21:37:10 -0000	1.19
--- ui.html	28 Sep 2003 02:40:08 -0000	1.19.2.1
***************
*** 3,7 ****
  
  Spambayes User Interface
- 
  
  
--- 3,6 ----



From anadelonbrin at users.sourceforge.net  Sat Sep 27 22:40:45 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sat Sep 27 22:40:47 2003
Subject: [Spambayes-checkins] spambayes/spambayes/resources ui.html, 1.20,
	1.21
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes/resources
In directory sc8-pr-cvs1:/tmp/cvs-serv11653/spambayes/resources

Modified Files:
	ui.html 
Log Message:
Anthony says pragma: no_cache is a no-no, so away it goes.

Index: ui.html
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/resources/ui.html,v
retrieving revision 1.20
retrieving revision 1.21
diff -C2 -d -r1.20 -r1.21
*** ui.html	19 Sep 2003 23:38:10 -0000	1.20
--- ui.html	28 Sep 2003 02:40:43 -0000	1.21
***************
*** 3,7 ****
  
  Spambayes User Interface
- 
  
  
--- 3,6 ----



From anadelonbrin at users.sourceforge.net  Sun Sep 28 19:55:28 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sun Sep 28 19:55:32 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_imapfilter.py,1.7,1.8
Message-ID: 

Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv31504/scripts

Modified Files:
	sb_imapfilter.py 
Log Message:
Fix a bug reported on the list by Matt Stegman, where trying to
view imap folders before restarting would cause a traceback.

Index: sb_imapfilter.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_imapfilter.py,v
retrieving revision 1.7
retrieving revision 1.8
diff -C2 -d -r1.7 -r1.8
*** sb_imapfilter.py	9 Sep 2003 08:07:10 -0000	1.7
--- sb_imapfilter.py	28 Sep 2003 23:55:26 -0000	1.8
***************
*** 798,802 ****
              imap = IMAPSession(server, port, imapDebug, doExpunge)
          httpServer = UserInterfaceServer(options["html_ui", "port"])
!         httpServer.register(IMAPUserInterface(classifier, imap, pwd))
          Dibbler.run(launchBrowser=launchUI)
      else:
--- 798,803 ----
              imap = IMAPSession(server, port, imapDebug, doExpunge)
          httpServer = UserInterfaceServer(options["html_ui", "port"])
!         httpServer.register(IMAPUserInterface(classifier, imap, pwd,
!                                               IMAPSession))
          Dibbler.run(launchBrowser=launchUI)
      else:



From anadelonbrin at users.sourceforge.net  Sun Sep 28 19:55:29 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sun Sep 28 19:55:35 2003
Subject: [Spambayes-checkins] spambayes/spambayes ImapUI.py,1.20,1.21
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv31504/spambayes

Modified Files:
	ImapUI.py 
Log Message:
Fix a bug reported on the list by Matt Stegman, where trying to
view imap folders before restarting would cause a traceback.

Index: ImapUI.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/ImapUI.py,v
retrieving revision 1.20
retrieving revision 1.21
diff -C2 -d -r1.20 -r1.21
*** ImapUI.py	24 Sep 2003 05:28:53 -0000	1.20
--- ImapUI.py	28 Sep 2003 23:55:26 -0000	1.21
***************
*** 45,49 ****
  
  import UserInterface
! from Options import options, optionsPathname
  
  # These are the options that will be offered on the configuration page.
--- 45,49 ----
  
  import UserInterface
! from spambayes.Options import options, optionsPathname
  
  # These are the options that will be offered on the configuration page.
***************
*** 107,111 ****
  class IMAPUserInterface(UserInterface.UserInterface):
      """Serves the HTML user interface for the proxies."""
!     def __init__(self, cls, imap, pwd):
          global parm_map
          # Only offer SSL if it is available
--- 107,111 ----
  class IMAPUserInterface(UserInterface.UserInterface):
      """Serves the HTML user interface for the proxies."""
!     def __init__(self, cls, imap, pwd, imap_session_class):
          global parm_map
          # Only offer SSL if it is available
***************
*** 122,125 ****
--- 122,126 ----
          self.imap_logged_in = False
          self.app_for_version = "IMAP Filter"
+         self.imap_session_class = imap_session_class
  
      def onHome(self):
***************
*** 195,199 ****
                  else:
                      port = 143
!             imap = IMAPSession(server, port)
          if self.imap is None:
              content = self._buildBox("Error", None,
--- 196,200 ----
                  else:
                      port = 143
!             imap = self.imap_session_class(server, port)
          if self.imap is None:
              content = self._buildBox("Error", None,



From anadelonbrin at users.sourceforge.net  Sun Sep 28 19:57:15 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sun Sep 28 19:57:18 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_imapfilter.py, 1.7,
	1.7.2.1
Message-ID: 

Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv31820/scripts

Modified Files:
      Tag: release_1_0
	sb_imapfilter.py 
Log Message:
Fix a bug reported on the list by Matt Stegman, where trying to
view imap folders before restarting would cause a traceback.

Index: sb_imapfilter.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_imapfilter.py,v
retrieving revision 1.7
retrieving revision 1.7.2.1
diff -C2 -d -r1.7 -r1.7.2.1
*** sb_imapfilter.py	9 Sep 2003 08:07:10 -0000	1.7
--- sb_imapfilter.py	28 Sep 2003 23:57:12 -0000	1.7.2.1
***************
*** 798,802 ****
              imap = IMAPSession(server, port, imapDebug, doExpunge)
          httpServer = UserInterfaceServer(options["html_ui", "port"])
!         httpServer.register(IMAPUserInterface(classifier, imap, pwd))
          Dibbler.run(launchBrowser=launchUI)
      else:
--- 798,803 ----
              imap = IMAPSession(server, port, imapDebug, doExpunge)
          httpServer = UserInterfaceServer(options["html_ui", "port"])
!         httpServer.register(IMAPUserInterface(classifier, imap, pwd,
!                                               IMAPSession))
          Dibbler.run(launchBrowser=launchUI)
      else:



From anadelonbrin at users.sourceforge.net  Sun Sep 28 19:57:15 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sun Sep 28 19:57:21 2003
Subject: [Spambayes-checkins] spambayes/spambayes ImapUI.py, 1.18.2.1,
	1.18.2.2
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv31820/spambayes

Modified Files:
      Tag: release_1_0
	ImapUI.py 
Log Message:
Fix a bug reported on the list by Matt Stegman, where trying to
view imap folders before restarting would cause a traceback.

Index: ImapUI.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/ImapUI.py,v
retrieving revision 1.18.2.1
retrieving revision 1.18.2.2
diff -C2 -d -r1.18.2.1 -r1.18.2.2
*** ImapUI.py	24 Sep 2003 03:54:15 -0000	1.18.2.1
--- ImapUI.py	28 Sep 2003 23:57:12 -0000	1.18.2.2
***************
*** 105,109 ****
  class IMAPUserInterface(UserInterface.UserInterface):
      """Serves the HTML user interface for the proxies."""
!     def __init__(self, cls, imap, pwd):
          global parm_map
          # Only offer SSL if it is available
--- 105,109 ----
  class IMAPUserInterface(UserInterface.UserInterface):
      """Serves the HTML user interface for the proxies."""
!     def __init__(self, cls, imap, pwd, imap_session_class):
          global parm_map
          # Only offer SSL if it is available
***************
*** 120,123 ****
--- 120,124 ----
          self.imap_logged_in = False
          self.app_for_version = "IMAP Filter"
+         self.imap_session_class = imap_session_class
  
      def onHome(self):
***************
*** 190,194 ****
                  else:
                      port = 143
!             imap = IMAPSession(server, port)
          if self.imap is None:
              content = self._buildBox("Error", None,
--- 191,195 ----
                  else:
                      port = 143
!             imap = self.imap_session_class(server, port)
          if self.imap is None:
              content = self._buildBox("Error", None,



From anadelonbrin at users.sourceforge.net  Sun Sep 28 20:02:51 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sun Sep 28 20:02:54 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_smtpproxy.py,1.3,1.3.2.1
Message-ID: 

Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv32724/scripts

Modified Files:
      Tag: release_1_0
	sb_smtpproxy.py 
Log Message:
Fix a print error reported by Austine Jane on the list.

Index: sb_smtpproxy.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/Attic/sb_smtpproxy.py,v
retrieving revision 1.3
retrieving revision 1.3.2.1
diff -C2 -d -r1.3 -r1.3.2.1
*** sb_smtpproxy.py	19 Sep 2003 07:51:06 -0000	1.3
--- sb_smtpproxy.py	29 Sep 2003 00:02:49 -0000	1.3.2.1
***************
*** 443,448 ****
          if not self.train_message_in_pop3proxy_cache(id, isSpam) and \
             not self.train_message_on_imap_server(id, isSpam):
!             print "Could not find message (%s); perhaps it was " + \
!                   "deleted from the POP3Proxy cache or the IMAP " + \
                    "server.  This means that no training was done." % (id, )
  
--- 443,448 ----
          if not self.train_message_in_pop3proxy_cache(id, isSpam) and \
             not self.train_message_on_imap_server(id, isSpam):
!             print "Could not find message (%s); perhaps it was " \
!                   "deleted from the POP3Proxy cache or the IMAP " \
                    "server.  This means that no training was done." % (id, )
  



From anadelonbrin at users.sourceforge.net  Sun Sep 28 20:03:21 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sun Sep 28 20:03:23 2003
Subject: [Spambayes-checkins] spambayes/spambayes smtpproxy.py,1.1,1.2
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv346/spambayes

Modified Files:
	smtpproxy.py 
Log Message:
Fix a print error reported by Austine Jane on the list.

Index: smtpproxy.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/smtpproxy.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** smtpproxy.py	19 Sep 2003 23:38:10 -0000	1.1
--- smtpproxy.py	29 Sep 2003 00:03:19 -0000	1.2
***************
*** 435,440 ****
          if not self.train_message_in_pop3proxy_cache(id, isSpam) and \
             not self.train_message_on_imap_server(id, isSpam):
!             print "Could not find message (%s); perhaps it was " + \
!                   "deleted from the POP3Proxy cache or the IMAP " + \
                    "server.  This means that no training was done." % (id, )
  
--- 435,440 ----
          if not self.train_message_in_pop3proxy_cache(id, isSpam) and \
             not self.train_message_on_imap_server(id, isSpam):
!             print "Could not find message (%s); perhaps it was " \
!                   "deleted from the POP3Proxy cache or the IMAP " \
                    "server.  This means that no training was done." % (id, )
  



From mhammond at users.sourceforge.net  Sun Sep 28 22:14:28 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Sun Sep 28 22:14:31 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs/resources
	dialogs.h, 1.20, 1.21 dialogs.rc, 1.39, 1.40
Message-ID: 

Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs/resources
In directory sc8-pr-cvs1:/tmp/cvs-serv20556/dialogs/resources

Modified Files:
	dialogs.h dialogs.rc 
Log Message:
Add slightly better stats, and a better framework to extend.


Index: dialogs.h
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/resources/dialogs.h,v
retrieving revision 1.20
retrieving revision 1.21
diff -C2 -d -r1.20 -r1.21
*** dialogs.h	5 Sep 2003 06:51:00 -0000	1.20
--- dialogs.h	29 Sep 2003 02:14:26 -0000	1.21
***************
*** 97,100 ****
--- 97,102 ----
  #define IDC_WIZ_GRAPHIC                 1092
  #define IDC_BUT_VIEW_LOG                1093
+ #define IDC_EDIT1                       1094
+ #define IDC_STATISTICS                  1095
  
  // Next default values for new objects
***************
*** 104,108 ****
  #define _APS_NEXT_RESOURCE_VALUE        128
  #define _APS_NEXT_COMMAND_VALUE         40001
! #define _APS_NEXT_CONTROL_VALUE         1094
  #define _APS_NEXT_SYMED_VALUE           101
  #endif
--- 106,110 ----
  #define _APS_NEXT_RESOURCE_VALUE        128
  #define _APS_NEXT_COMMAND_VALUE         40001
! #define _APS_NEXT_CONTROL_VALUE         1096
  #define _APS_NEXT_SYMED_VALUE           101
  #endif

Index: dialogs.rc
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/resources/dialogs.rc,v
retrieving revision 1.39
retrieving revision 1.40
diff -C2 -d -r1.39 -r1.40
*** dialogs.rc	17 Sep 2003 00:11:44 -0000	1.39
--- dialogs.rc	29 Sep 2003 02:14:26 -0000	1.40
***************
*** 48,55 ****
                      IDC_INBOX_TIMER_ONLY,"Button",BS_AUTOCHECKBOX | 
                      WS_TABSTOP,16,100,217,10
!     PUSHBUTTON      "Show Data Folder",IDC_SHOW_DATA_FOLDER,16,127,70,14
      CONTROL         "Enable background filtering",IDC_BUT_TIMER_ENABLED,
                      "Button",BS_AUTOCHECKBOX | WS_TABSTOP,16,12,162,10
!     PUSHBUTTON      "Diagnostics...",IDC_BUT_SHOW_DIAGNOSTICS,163,126,70,14
  END
  
--- 48,58 ----
                      IDC_INBOX_TIMER_ONLY,"Button",BS_AUTOCHECKBOX | 
                      WS_TABSTOP,16,100,217,10
!     PUSHBUTTON      "Show Data Folder",IDC_SHOW_DATA_FOLDER,7,190,70,14
      CONTROL         "Enable background filtering",IDC_BUT_TIMER_ENABLED,
                      "Button",BS_AUTOCHECKBOX | WS_TABSTOP,16,12,162,10
!     PUSHBUTTON      "Diagnostics...",IDC_BUT_SHOW_DIAGNOSTICS,171,190,70,14
!     GROUPBOX        "Statistics",IDC_STATIC,7,125,234,58
!     LTEXT           "some stats\nand some more\nline 3\nline 4\nline 5",
!                     IDC_STATISTICS,12,134,223,43,SS_SUNKEN
  END
  



From mhammond at users.sourceforge.net  Sun Sep 28 22:14:28 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Sun Sep 28 22:14:34 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs dialog_map.py,
	1.34, 1.35
Message-ID: 

Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs
In directory sc8-pr-cvs1:/tmp/cvs-serv20556/dialogs

Modified Files:
	dialog_map.py 
Log Message:
Add slightly better stats, and a better framework to extend.


Index: dialog_map.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/dialog_map.py,v
retrieving revision 1.34
retrieving revision 1.35
diff -C2 -d -r1.34 -r1.35
*** dialog_map.py	16 Sep 2003 23:51:12 -0000	1.34
--- dialog_map.py	29 Sep 2003 02:14:26 -0000	1.35
***************
*** 16,19 ****
--- 16,27 ----
  
  # "dialog specific" processors:
+ class StatsProcessor(ControlProcessor):
+     def Init(self):
+         text = "\n".join(self.window.manager.stats.GetStats())
+         win32gui.SendMessage(self.GetControl(), win32con.WM_SETTEXT, 0, text)
+ 
+     def GetPopupHelpText(self, cid):
+         return "Displays statistics on mail processed by SpamBayes"
+ 
  class VersionStringProcessor(ControlProcessor):
      def Init(self):
***************
*** 475,478 ****
--- 483,487 ----
          (EditNumberProcessor,   "IDC_DELAY2_TEXT IDC_DELAY2_SLIDER", "Filter.timer_interval", 0, 10, 20),
          (BoolButtonProcessor,   "IDC_INBOX_TIMER_ONLY", "Filter.timer_only_receive_folders"),
+         (StatsProcessor,        "IDC_STATISTICS"),
          (CommandButtonProcessor,  "IDC_SHOW_DATA_FOLDER", ShowDataFolder, ()),
          (DialogCommand,         "IDC_BUT_SHOW_DIAGNOSTICS", "IDD_DIAGNOSTIC"),



From mhammond at users.sourceforge.net  Sun Sep 28 22:14:28 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Sun Sep 28 22:14:37 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000 oastats.py, NONE,
	1.1 addin.py, 1.112, 1.113 filter.py, 1.33, 1.34 manager.py,
	1.87, 1.88
Message-ID: 

Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv20556

Modified Files:
	addin.py filter.py manager.py 
Added Files:
	oastats.py 
Log Message:
Add slightly better stats, and a better framework to extend.


--- NEW FILE: oastats.py ---
# oastats.py - Outlook Addin Stats

class Stats:
    def __init__(self, config):
        self.config = config
        self.Reset()
    def Reset(self):
        self.num_ham = self.num_spam = self.num_unsure = 0
        self.num_deleted_spam = self.num_deleted_spam_fn  = 0
        self.num_recovered_good = self.num_recovered_good_fp = 0
    def RecordClassification(self, score):
        score *= 100 # same units as our config values.
        if score >= self.config.filter.spam_threshold:
            self.num_spam += 1
        elif score >= self.config.filter.unsure_threshold:
            self.num_unsure += 1
        else:
            self.num_ham += 1
    def RecordManualClassification(self, recover_as_good, score):
        score *= 100 # same units as our config values.
        if recover_as_good:
            self.num_recovered_good += 1
            # If we are recovering an item that is in the "spam" threshold,
            # then record it as a "false positive"
            if score > self.config.filter.spam_threshold:
                self.num_recovered_good_fp += 1
        else:
            self.num_deleted_spam += 1
            # If we are deleting as Spam an item that was in our "good" range,
            # then record it as a false neg.
            if score < self.config.filter.unsure_threshold:
                self.num_deleted_spam_fn += 1
    def GetStats(self):
        num_seen = self.num_ham + self.num_spam + self.num_unsure
        if num_seen==0:
            return ["SpamBayes has processed zero messages"]
        chunks = []
        push = chunks.append
        perc_ham = 100.0 * self.num_ham / num_seen
        perc_spam = 100.0 * self.num_spam / num_seen
        perc_unsure = 100.0 * self.num_unsure / num_seen
        format_dict = dict(perc_spam=perc_spam, perc_ham=perc_ham,
                           perc_unsure=perc_unsure, num_seen = num_seen)
        format_dict.update(self.__dict__)
        push("SpamBayes has processed %(num_seen)d messages - " \
             "%(num_ham)d (%(perc_ham)d%%) good, " \
             "%(num_spam)d (%(perc_spam)d%%) spam " \
             "and %(num_unsure)d (%(perc_unsure)d%%) unsure" % format_dict)
        if self.num_recovered_good:
            push("%(num_recovered_good)d message(s) were manually " \
                 "classified as good (with %(num_recovered_good_fp)d " \
                 "being false positives)" % format_dict)
        else:
            push("No messages were manually classified as good")
        if self.num_deleted_spam:
            push("%(num_deleted_spam)d message(s) were manually " \
                 "classified as spam (with %(num_deleted_spam_fn)d " \
                 "being false negatives)" % format_dict)
        else:
            push("No messages were manually classified as spam")
        return chunks

if __name__=='__main__':
    class FilterConfig:
        unsure_threshold = 15
        spam_threshold = 85
    class Config:
        filter = FilterConfig()
    # processed zero
    s = Stats(Config())
    print "\n".join(s.GetStats())
    # No recovery
    s = Stats(Config())
    s.RecordClassification(.2)
    print "\n".join(s.GetStats())
    
    s = Stats(Config())
    s.RecordClassification(.2)
    s.RecordClassification(.1)
    s.RecordClassification(.4)
    s.RecordClassification(.9)
    s.RecordManualClassification(True, 0.1)
    s.RecordManualClassification(True, 0.9)
    s.RecordManualClassification(False, 0.1)
    s.RecordManualClassification(False, 0.9)
    print "\n".join(s.GetStats())

Index: addin.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/addin.py,v
retrieving revision 1.112
retrieving revision 1.113
diff -C2 -d -r1.112 -r1.113
*** addin.py	15 Sep 2003 06:26:35 -0000	1.112
--- addin.py	29 Sep 2003 02:14:25 -0000	1.113
***************
*** 611,615 ****
          if not self.manager.config.filter.enabled:
              self.manager.ReportError(
!                 "You must enable SpamBayes before you can delete as spam")
              return
          SetWaitCursor(1)
--- 611,616 ----
          if not self.manager.config.filter.enabled:
              self.manager.ReportError(
!                 "You must configure and enable SpamBayes before you can " \
!                 "delete as spam")
              return
          SetWaitCursor(1)
***************
*** 631,634 ****
--- 632,638 ----
          new_msg_state = self.manager.config.general.delete_as_spam_message_state
          for msgstore_message in msgstore_messages:
+             # Record this recovery in our stats.
+             self.manager.stats.RecordManualClassification(False,
+                                     self.manager.score(msgstore_message))
              # Must train before moving, else we lose the message!
              subject = msgstore_message.GetSubject()
***************
*** 666,670 ****
          if not self.manager.config.filter.enabled:
              self.manager.ReportError(
!                 "You must enable SpamBayes before you can recover spam")
              return
          SetWaitCursor(1)
--- 670,675 ----
          if not self.manager.config.filter.enabled:
              self.manager.ReportError(
!                 "You must configure and enable SpamBayes before you can " \
!                 "recover spam")
              return
          SetWaitCursor(1)
***************
*** 680,683 ****
--- 685,691 ----
              # that the source folder == dest folder - restore to
              # the inbox in this case.
+             # (But more likely is that the original store may be read-only
+             # so we were unable to record the initial folder, as we save it
+             # *before* we do the move (and saving after is hard))
              try:
                  subject = msgstore_message.GetSubject()
***************
*** 688,691 ****
--- 696,702 ----
                      restore_folder = inbox_folder
  
+                 # Record this recovery in our stats.
+                 self.manager.stats.RecordManualClassification(True,
+                                         self.manager.score(msgstore_message))
                  # Must train before moving, else we lose the message!
                  print "Recovering to folder '%s' and ham training message '%s' - " % (restore_folder.name, subject),
***************
*** 1235,1239 ****
  
      def ProcessMissedMessages(self):
-         # This could possibly spawn threads if it was too slow!
          from time import clock
          config = self.manager.config.filter
--- 1246,1249 ----
***************
*** 1339,1345 ****
              # it (ie, the dialog)
              self.manager.Save()
!             stats = self.manager.stats
!             print "SpamBayes processed %d messages, finding %d spam and %d unsure" % \
!                 (stats.num_seen, stats.num_spam, stats.num_unsure)
              self.manager.Close()
              self.manager = None
--- 1349,1354 ----
              # it (ie, the dialog)
              self.manager.Save()
!             # Report some simple stats.
!             print "\r\n".join(self.manager.stats.GetStats())
              self.manager.Close()
              self.manager = None

Index: filter.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/filter.py,v
retrieving revision 1.33
retrieving revision 1.34
diff -C2 -d -r1.33 -r1.34
*** filter.py	19 Sep 2003 04:03:38 -0000	1.33
--- filter.py	29 Sep 2003 02:14:25 -0000	1.34
***************
*** 13,17 ****
      config = mgr.config.filter
      prob = mgr.score(msg)
-     mgr.stats.num_seen += 1
      prob_perc = prob * 100
      if prob_perc >= config.spam_threshold:
--- 13,16 ----
***************
*** 81,84 ****
--- 80,84 ----
                  raise RuntimeError, "Eeek - bad action '%r'" % (action,)
  
+         mgr.stats.RecordClassification(prob)
          return disposition
      except:

Index: manager.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/manager.py,v
retrieving revision 1.87
retrieving revision 1.88
diff -C2 -d -r1.87 -r1.88
*** manager.py	10 Sep 2003 07:42:45 -0000	1.87
--- manager.py	29 Sep 2003 02:14:25 -0000	1.88
***************
*** 15,18 ****
--- 15,19 ----
  
  import msgstore
+ import oastats
  
  try:
***************
*** 133,139 ****
      pass
  
- class Stats:
-     def __init__(self):
-         self.num_seen = self.num_spam = self.num_unsure = 0
  
  # Function to "safely" save a pickle, only overwriting
--- 134,137 ----
***************
*** 323,327 ****
          self.addin = None
          self.verbose = verbose
-         self.stats = Stats()
          self.outlook = outlook
          self.dialog_parser = None
--- 321,324 ----
***************
*** 386,389 ****
--- 383,387 ----
          self.classifier_data = ClassifierData(db_manager, self)
          self.LoadBayes()
+         self.stats = oastats.Stats(self.config)
  
      # "old" bayes functions - new code should use "classifier_data" directly



From mhammond at users.sourceforge.net  Sun Sep 28 22:18:29 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Sun Sep 28 22:18:31 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_service.py, 1.14,
	1.15
Message-ID: 

Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv21240

Modified Files:
	pop3proxy_service.py 
Log Message:
Don't start the service if a proxy is already running.


Index: pop3proxy_service.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_service.py,v
retrieving revision 1.14
retrieving revision 1.15
diff -C2 -d -r1.14 -r1.15
*** pop3proxy_service.py	18 Sep 2003 04:15:24 -0000	1.14
--- pop3proxy_service.py	29 Sep 2003 02:18:27 -0000	1.15
***************
*** 105,110 ****
  
      def SvcDoRun(self):
          # Setup our state etc
!         sb_server.prepare(state=sb_server.state)
          assert not sb_server.state.launchUI, "Service can't launch a UI"
  
--- 105,120 ----
  
      def SvcDoRun(self):
+         import servicemanager
          # Setup our state etc
!         try:
!             sb_server.prepare(state=sb_server.state)
!         except sb_server.AlreadyRunningException:
!             msg = "The SpamBayes proxy service could not be started, as "\
!                   "another SpamBayes server is already running on this machine"
!             servicemanager.LogErrorMsg(msg)
!             errCode = winerror.ERROR_SERVICE_SPECIFIC_ERROR 
!             self.ReportServiceStatus(win32service.SERVICE_STOPPED, 
!                                      win32ExitCode=errCode, svcExitCode = 1)
!             return
          assert not sb_server.state.launchUI, "Service can't launch a UI"
  
***************
*** 119,123 ****
                  % (win32api.GetUserName(),
                     optionsPathname)
-         import servicemanager
          servicemanager.LogMsg(
              servicemanager.EVENTLOG_INFORMATION_TYPE,
--- 129,132 ----



From anadelonbrin at users.sourceforge.net  Mon Sep 29 00:43:11 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Mon Sep 29 00:43:15 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_server.py,1.9,1.10
Message-ID: 

Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv9589/scripts

Modified Files:
	sb_server.py 
Log Message:
Add space for a warning message to the status box on the home page
of the web interface.

Add the same warnings that the Outlook plug-in uses to it - i.e. warn
if the user has unbalanced ham and spam, or not much ham and
spam, or zero ham and spam.

Index: sb_server.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_server.py,v
retrieving revision 1.9
retrieving revision 1.10
diff -C2 -d -r1.9 -r1.10
*** sb_server.py	25 Sep 2003 00:10:31 -0000	1.9
--- sb_server.py	29 Sep 2003 04:43:09 -0000	1.10
***************
*** 700,703 ****
--- 700,730 ----
          self.bayes = storage.open_storage(filename, self.useDB)
  
+         # Status message
+         nspam = self.bayes.nspam
+         nham = self.bayes.nham
+         if nspam > 10 and nham > 10:
+             db_ratio = nham/float(nspam)
+             big = small = None
+             if db_ratio > 5.0:
+                 big = "ham"
+                 small = "spam"
+             elif db_ratio < (1/5.0):
+                 big = "spam"
+                 small = "ham"
+             if big is not None:                
+                 self.warning = "%s\nWarning: you have much more %s than %s - " \
+                                "SpamBayes works best with approximately even " \
+                                "numbers of ham and spam." % (db_status, big,
+                                                              small)
+             else:
+                 self.warning = ""
+         elif nspam > 0 or nham > 0:
+             self.warning = "Database only has %d good and %d spam - you should " \
+                            "consider performing additional training." % (nham, nspam)
+         else:
+             self.warning = "Database has no training information.  SpamBayes " \
+                            "will classify all messages as 'unsure', " \
+                            "ready for you to train."
+ 
          # Don't set up the caches and training objects when running the self-test,
          # so as not to clutter the filesystem.



From anadelonbrin at users.sourceforge.net  Mon Sep 29 00:43:11 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Mon Sep 29 00:43:18 2003
Subject: [Spambayes-checkins] spambayes/spambayes ImapUI.py,1.21,1.22
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv9589/spambayes

Modified Files:
	ImapUI.py 
Log Message:
Add space for a warning message to the status box on the home page
of the web interface.

Add the same warnings that the Outlook plug-in uses to it - i.e. warn
if the user has unbalanced ham and spam, or not much ham and
spam, or zero ham and spam.

Index: ImapUI.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/ImapUI.py,v
retrieving revision 1.21
retrieving revision 1.22
diff -C2 -d -r1.21 -r1.22
*** ImapUI.py	28 Sep 2003 23:55:26 -0000	1.21
--- ImapUI.py	29 Sep 2003 04:43:09 -0000	1.22
***************
*** 127,130 ****
--- 127,131 ----
          """Serve up the homepage."""
          stateDict = self.classifier.__dict__.copy()
+         stateDict["warning"] = ""
          stateDict.update(self.classifier.__dict__)
          statusTable = self.html.statusTable.clone()



From anadelonbrin at users.sourceforge.net  Mon Sep 29 00:43:11 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Mon Sep 29 00:43:23 2003
Subject: [Spambayes-checkins] spambayes/spambayes/resources ui.html, 1.21,
	1.22 ui_html.py, 1.21, 1.22
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes/resources
In directory sc8-pr-cvs1:/tmp/cvs-serv9589/spambayes/resources

Modified Files:
	ui.html ui_html.py 
Log Message:
Add space for a warning message to the status box on the home page
of the web interface.

Add the same warnings that the Outlook plug-in uses to it - i.e. warn
if the user has unbalanced ham and spam, or not much ham and
spam, or zero ham and spam.

Index: ui.html
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/resources/ui.html,v
retrieving revision 1.21
retrieving revision 1.22
diff -C2 -d -r1.21 -r1.22
*** ui.html	28 Sep 2003 02:40:43 -0000	1.21
--- ui.html	29 Sep 2003 04:43:09 -0000	1.22
***************
*** 145,148 ****
--- 145,153 ----
          
                		
      
                
+             Warning: please insert warning message here!
+         
                
      
                
  

Index: ui_html.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/resources/ui_html.py,v
retrieving revision 1.21
retrieving revision 1.22
diff -C2 -d -r1.21 -r1.22
*** ui_html.py	19 Sep 2003 23:38:10 -0000	1.21
--- ui_html.py	29 Sep 2003 04:43:09 -0000	1.22
***************
*** 6,100 ****
  
  import zlib
! data = zlib.decompress("x\\{s6?U\016\010S6Zi#d$ΖX'M6JT\030\0223\025η\033\000\011>\
! a;U\000\032ݍ\037\033ξp1\015Ǯz\005s\016ß/ɥzq2Fl\
! $H&<\036\016_u(1\026_r\005\022Z'ŹÀ\007\013ao\013ro\036(\023c(\017\030\010\037l\006\024{\031Y\
! \003~\020\005X/NY\031\015pj޼\001)ĒGɅrgGKhVz>[o\021\
! Q0gz\002Rr6b\000@¯\003\026Y\022`<\011\017gBĊћ\022W<\
! Ux+2@\001c\023x\000/\\\021H#\034{\032\013aM\004,\023\"\001\0328\014\014Di\\Ɯ\036\024DV&E\004\
! Ē,|\002J\022OLͦ NvE/_1r2eJ?`@,B\030VU@.Ss\
! \020Ni齈wZ,R[\002?\010\0060\020\003\"`\003\007$MS\030T%\007@6^*\
! <\002\035(\032o`-WڬI\022!\034u3\025J}Z{\013\024*ц@|Fa\003T8(~*\026\000\";>8\
! \033AB0k|B\024+j\017\032VaTL#\010B\021m^ziB\007JYeAP\
! 7qX\012~\034\024k!;#\031jת$Ij\023ig|2qCzJ?\
! *KA\015qߊWE@SuGU|\020_x\017Ά\006l\0162+,vΙLEh0p\
! \016\025>,d<ߑR(\003 T30@\002\022\036\022ݕL\001 A=fqq?\024o2!\0020w\
! \027b@@Fb\010C\014\013m`gC$֮>E8QFW&߂v\\r]䉫\027\021\006\
! ;8|*\005\027F\037\034\007J\003&35\000%5*jL.Y\036d\000<\024[\006!\004*\
! \007p4\001?}2\012\032\005'\034\034\013!\003V`\010ގ@\\7\032@8(8\"\"\000b\"\
! V5ܶtH(V\\Tܠ&pM\011?\001-BZ+\013ZH\003\023\033FY/\010\030\012d\
! &P#>ǧ(|jd\005bcI:\005\"gY[cVg\020\032ri׏]\036\034\
! =\030f\004E\010^\004t:\024\010\014\002u#(,Dm\000U\031\0100P\037DͼExSw\
! <\\X0k\023tZ1\036ijAVAa\031.R͆w\007\
! U30[esr\017\035PDͽIi`Ợ$\036.!F\017nj^b \
! Φ~5-F#Ol8\017酶\004fIb٬W\021\026yy\012A(n\023\
! \024f>n=&N\002Rh6σ\000]\012s{0,x(b%Nh\026ᲵIĬ\
! 5 nW^~J2\023n%=l\0139A}0$E\014ӧ$5\006rޤshؤn\013\002\021\
! tk?/ţAc=LU\012\000\003EMl*\004X6r\031;Lf]\036?\\!\002\
! &$0{$]HOy!:Pi\031\013%\0065_\"P%AgflZh[V\025&\020w6\003\027R\
! dhpҨ\001(أr^7\010]\001n(rʹ)35|\026GbEk!YVbow\
! \036uDʪ֦\035#%-(*?VR\015HB\026XPUMC;+OZRX\021+uy\032\026싅\010I42!\
! tZ\026\005b\006\022\000\021?\012X;A.d\020b\0021RDB\037tyG+Fuŕ@\0154\
! 1\035&^h\035QNQ\036,dCF7\010D#`\0114\003Uo%I\017@\011]g\\\025\033\001\
! r}+(Y5*\000e\037\005\032ΦьiK\023/1\001q>\024x3I\"(?s\027Q\030B\\Űw\
! tI7FReng\003\006\002\011x꯮zG9\021טnhn:\015\016r\012\\Zr*\
! Osw%d\034\0020ʑ? 1JJa.\\o\003,qp}x݄\021Z~떂v*э\
! -\033t\015IA\026F,\027\014\037z7^\\Ue\\N&\012'mA6>f'\
! P\006\022\001Y \026\034T\035]?6H^vN3\027ACE쌺\001_\
! l;U9$'T\007\"\001^cwg\010\005z\0321[\000~!j\015\031SW\
! +\016\012NDv\001~\004\011U\001a\024=2Ⱥ՗pou\036FƲ#\032^~;\011sM\
! Y@6\000Ux\021o˩YDpCp7\013\025Q1ůPY]S\005jS;\
! U\013Sn:ONX0&\006!\032jDߍ蕘v\033_Z)\035\004\002\022n\
! VlD\033=+Ze'r\024\026\011\007N\015\032\\\013\000\"#\025\000\0148Xq\
! y(\"Zx\005 \":\013`,Fd\036BV]`\010 },H\031N6H_TU[\005\0227r\
! U6xo˷МUB3jX\030ܛ(XD]EI\010 E;}\020'XR~\
! \017glvqSZW\026&1\010~@j\0131I#`Z'\030[\036n\035\037\033ƾ;]HT\
! a\0211/\016hnEʎ<\037{Ѩ\010\001\001\02693S\004\031\017#i@ax&8\0253Yv`\
! ObE\035Ū\012؍\006ѱ\016g\017V&um'F?\021ڊ\022\014ꋠN\005PM\017 T\
! G9V;jj\0333V1-xW\015&X.\031\035\0046[\007g] K$ctt\
! RMjҢ\"t\032Bיi\024G׮3J:**SȠI!یCV1u|o_3`Ϸ6d\
! {6eʣ\001\011]s7Wr!8!8dd,8oz\021:S1$e\022D\
! 5s}z\0124x\0041^'d)\\Ҷ\015uUƻKEà]zIo\
! \014dywurc=C\037!\005U*\002N4S\001.!Y{Ur\010Pf9Xvʣ\025Z_:{P<\024\
! J}\011G':q\003ۼ0_W~cJg\000yX:hNxf\025)NAt_\036t\003\021}\"\
! \"C\016C}}Gd.\017dwo6\037\013|\0340{AD\0139jD-Km\016yc\037OXW\
! a;e6~`2lasEoǎ^\006\020oR- :[i\032ڃC'q\012\035\
! ԛ<ϰ%\016n\036OZB\025tb(Pj/\\SjQcL{\0354\
! $\001=&U&n\017P.\025\032:Ĕ\016\0270[}]W\002:jۖ5\0372D_\
! Yt9\021͐jd\"RC\"O|W]\034\015mH\020>84lLƀ\
! vN\010s\015\010t1EZ\027L\022Y\\ۣ|jK<}bHkk7\010RR|\
! W\021GT|l\022\027j\007刊nk#Xʄ;Č*Y;b©\033\020?P\
! #[ͨS3vlDީ(0W3Jg/\024eK%ۘO7\027e\021Uҷ㠾*jP\
! \0043\017Rӧ\017Q!OИ޿sQf\"\007\036`ֹX{w2lZ\
! r.\030(2h6X#@\027\0254Bn\022\002f\017\003_\035{lt2N\
! GxtH32h,t<\034VAU@\032l>\030:\023KCSI)\033V\015?0\033\034\
! rh3ci\0344pLg<\013\011eHZ\010v&'G__Z^")
  ### end
--- 6,101 ----
  
  import zlib
! data = zlib.decompress("x\\{s6?U\016\010S6Zi#d$ΖX'M֕JT\030\02235η\033\000\011>\
! GrU$>Fw\006.]L~~Ů&o߰_^y}px9T/N\006ވM2\
! Q\021Ʉ᫟\035o\0262߂X(\\|i\0144MeQe)\
! vK\014\021y\002\035\037?i]ky^,EPUB[ZL\001Q\\\004Cf\0256s\
! l\025\002\023#r\014d:f\\x=\032٪\0313x_li\003I/}Г\016\016\003֡4^\013\006\
! ;-\015V7Mn\003E&b(GEІrEet\005wg8Gޓӊ\017D_mgC\025gC\
! pIJD\013\0051sWy7y\012\010=\000\033\000\034,ZY\005z2G3\001`͓sOe\
! \024pd\007PhP\003Y0\0035\007K3\031A3_6\020\000OV8KSE&f0\020ܻ\
! \025<\033\022\0328ee4ez\006\0244C\022K\036%\027ʝ\035-[lqFGaZN\
! \011H\",\001\002\031\012\016X(fQ\002J(\026R~8Ix8\023\"V$ޔ\010FFGi[\001\012\017\030\
! {Y@\032\015؃d5\\\010kB&`\020L<\024\031\017a`\000$J2P$2)% d#(\024V\
! 8~dj6\005qB\005/|9㯜);VZ-\014\006\001\003bQ\024°\006r\004ԝpJ׏oE\
! \034\001JdR\022A`E6(4\034\030\026\001\017\037\000= i\"\036g,\007\017dS2#@ہ\012\
! rJ+\021\"Q7Sԧ@\022m\010g\024\032\013>? }O\005\010b\001 \003Z냳aj\033\013\012\
! ƗiL,J\007ZaV#*|\025X(\002\013P4M_i\037<,\035&\016+\036S\001\
! \032؏\003\026\027z0ĵsg9CZ1=RVmB0m/PF6nH^I\007x]{_\005q\
! )4[\010^}\010\012\035B\013\020߀́]f%؎9H\034\015\006\001sه\
! {;R\012e\000j\006\006\010TW\"CI3\025$6gL4.NM0D=\033F\"Y\\\014\032\
! \\,x2\027(a-\020쬳~h_dڵѧ\010'\010\026[ЎV\016[-4H\012\032|Y=(,EY\007/x+?M$\005S\001a8\036x\022I\035}\
! -U\0034Tw\033+\"\004{\011K]X1(E\006\006\004\017\025-ڧٰ\035`j\032fl\023\
! u^N\003\020j7:7\0157|Wu%dQhqK\022D\022ou\
! ;%thi}\015=ЖL6P,U\024*\002\"485^Q\013-x\002-\
! V@T\012-\035y\032KYuzo\032\005w\033=Q\011QV\024\"\\v\0345)5\006dp<\
! k/I^fխm!'\017&daԒ\0049\037@Λ}]\026\036mA Ÿtu\
! xp5h?[}l\034J\001`@h\011M\000FU\0252#3|\011լc篐+\004X`Є\004f\017Yd\
! \0131/\022\020^\007*-cĠFt\013[\004а0̌\"ǩ\023\015tZ\013X\"\0370\025\022\
! —&1\0039\027\000G\005ӭ2Zt\035Ru)P\011\004Ȋح_\032'\"e*|U\
! \000ո\000\020֪tr8jX\012\004 u\016qu^P=IQ17jHY-;rZPTZ\
! 0\032V\016kA⫢F\023\037C4\003p2& \016\007O&\026zb\011%\"\012C\010\030\026!].\011W_l\
! @ \001RUow\025@\0145\002/s\035XA^NQa8K4[n_\037iQ_Cf\020F9\
! \007$FV-\021\000*8܅\013m@6\016nt/\036\017?CO\025xZu==e#ү!)ȢԈ\005\
! ?=WO]8<\027\027@(\002=I;EH]TX\023^(r\011\000ӧ\
! d\020d\016yΎ\037Z$/;oumi\031ۋw\"vF\000/o6؝\034'ڄ\000\"\
! \001^cwg#\013\005z\0321[\000~!j\015\031S+\016\012FDv\001~\
! \004\011U\001a\024=2Ⱥ՗puHƲ#\032^~;KtMB%>\000Umɨ\021o\
! ˩YL\021PA\0205\013хxIʬL)\0065p)n`\020\004llH*P7WݱvЩ\
! sN\0304uQ\020\015A5M9v\033_Z9\035\004\0022nVlD\033\
! =+Ze'r\024\026\011GN\015\032\\\013\000\"#\025\000\0148Xqy(\"Zx\005 \":\013\
! `,Fd\036J[]\001\011 :@\014nj'\033h/nr\032$n.\004}o\0359\
! f\0121p]7Q\004\020@\022w%2zA``1\011\032|\014=\023e\"6\
! i9\003\022h\023DM\032\001:qJpoSLL\006\037cFZ\012yqF#v+Rvyl\
! ;\036F=Ub]\010\012r'\000x\030I\023\007EqL?\014DB\032fU\010G0p\
! Q),`I\016ep\026\003x,To%(xs\006\015\023zBVAb|@Bm\014lR\007J;8YJיfʥk\025zݚ\
! Y-duҢO(X[\001@wN\025\032׈|[q\021<+Ð\027ܬ\014\"\004U\0317e\
! \013kqC\0370{AD\033ͣ\032jD|n:Alh7YPMeqU\0074+\011c\
! \012Oe\013\004\004#ђ\007ʱIUs\027e\022e\001wcb\005i\024\030.ݥ\
! DrW\033\036n[3\026Y\004B'llcJ,ob;ўx\022D6\016u%\
! zۑeb)0WuhBJf\"{T=P\001\037\023\037i8Ճv'B;\014Ϟ=iFTS\
! \034i\015ڪo\030{5\001\007=\022 \004\027'-ѪК\032{͹'\001f3$\027\010y\036\
! \005F\032R\035r6;\003\013uW:6\021tB\012Ts0XLD.L+\000t%\036\021\
! TĎ\022oPt-,FqbZ\024Ӥ.&\031ͨF%\\\003\"u\001TS\003#H8UlՎZچLyo,\
! \025v\014xo\013^kI?)nrF'\015\036|\027\022\031\031\035\0359PUZ\010<{u\
! &|\032Q댒dk\030$2hR6UL\035\033\037\0141\020\036f\015mY\006q@B\
! \034U\\g\010\016d\010\0160Yd2%\026MO1ZGt9LU¿\027ӰfRO\006H\
! \010/1~^\035VNxw鳨\034?}\03045W_^1i>n5N񕴞\
! wsg#]wʑ%$q/]R.\025gی~RֹqGe\005l'Ԃ\
! >_|P\0000EN\000!Sw\022֍U~u\030?\001Q<{62֘Fr\015_\
! jT'\021~\005\032[\002F<]۲^\005S1@ +؊iw](h펣m\
! Th@\033QYR\037\010l~;T'tlj\013Q;S\005)\003\
! mRK߻N\014\005]T{˛ٺ~\\;+5&\024GX.qHs\030JMb\037oRUkV\
! x\016\005b\012]ILp\001׵k.mYS!3kMEeCH\036\014F\
! y-\"Z/5)\036~G\007nC3Vzd\014Xߊjt0܀M\027S^\
! u1z\014:,̵=Xͧ6\025CZ[A⻊U>Rfw?{ls\
! Wk\013.GTt]\033R&T!f\\U\021\026Nu<ިa\032߭ܚmFY,;\
! f#N}G&\027>k|(ӟR'[(P\035\007UQ:'Q~2>)C\
! \017#|\017\0052[\0279\030O\000}\015zޓ݌޿Z-ESwuDAi\032\
! a\027\035\007r\025e\037'և~\026\0205;[\030R\004cw2>>£sGA\007=fQ\
! pZ\015\002@f\010޷'ՙ^\032JjL1݈Oζj0Ut/IF)\036LSf:\
! Y\030ŭN(DrD3\031>r\002C\021o")
  ### end



From anadelonbrin at users.sourceforge.net  Mon Sep 29 01:09:02 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Mon Sep 29 01:09:05 2003
Subject: [Spambayes-checkins] spambayes/spambayes smtpproxy.py,1.2,1.3
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv13160/spambayes

Modified Files:
	smtpproxy.py 
Log Message:
Fix a bug reported by Austine Jane on the list.  If we successfully
trained a message from the pop3proxy cache or imap server, we
still said that we couldn't find it.

Index: smtpproxy.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/smtpproxy.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** smtpproxy.py	29 Sep 2003 00:03:19 -0000	1.2
--- smtpproxy.py	29 Sep 2003 05:09:00 -0000	1.3
***************
*** 456,459 ****
--- 456,460 ----
          targetCorpus.takeMessage(id, sourceCorpus)
          self.classifier.store()
+         return True
  
      def train_message_on_imap_server(self, id, isSpam):
***************
*** 473,476 ****
--- 474,479 ----
              self.classifier.learn(msg.asTokens(), isSpam)
              msg.RememberTrained(isSpam)
+         self.classifier.store()
+         return True
  
  def LoadServerInfo():



From anadelonbrin at users.sourceforge.net  Mon Sep 29 01:09:25 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Mon Sep 29 01:09:27 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_smtpproxy.py, 1.3.2.1,
	1.3.2.2
Message-ID: 

Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv13229/scripts

Modified Files:
      Tag: release_1_0
	sb_smtpproxy.py 
Log Message:
Fix a bug reported by Austine Jane on the list.  If we successfully
trained a message from the pop3proxy cache or imap server, we
still said that we couldn't find it.

Index: sb_smtpproxy.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/Attic/sb_smtpproxy.py,v
retrieving revision 1.3.2.1
retrieving revision 1.3.2.2
diff -C2 -d -r1.3.2.1 -r1.3.2.2
*** sb_smtpproxy.py	29 Sep 2003 00:02:49 -0000	1.3.2.1
--- sb_smtpproxy.py	29 Sep 2003 05:09:23 -0000	1.3.2.2
***************
*** 464,467 ****
--- 464,468 ----
          targetCorpus.takeMessage(id, sourceCorpus)
          self.classifier.store()
+         return True
  
      def train_message_on_imap_server(self, id, isSpam):
***************
*** 481,484 ****
--- 482,487 ----
              self.classifier.learn(msg.asTokens(), isSpam)
              msg.RememberTrained(isSpam)
+         self.classifier.store()
+         return True
  
  def LoadServerInfo():



From anadelonbrin at users.sourceforge.net  Mon Sep 29 03:26:30 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Mon Sep 29 03:26:33 2003
Subject: [Spambayes-checkins] spambayes/spambayes ProxyUI.py, 1.26,
	1.27 UserInterface.py, 1.26, 1.27
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv32479/spambayes

Modified Files:
	ProxyUI.py UserInterface.py 
Log Message:
Add a basic 'help' link to the web interface, which can be
a standard set of information, or specific to the page on
which the user clicked 'help'.  The actual help information
still needs to be added!

Index: ProxyUI.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/ProxyUI.py,v
retrieving revision 1.26
retrieving revision 1.27
diff -C2 -d -r1.26 -r1.27
*** ProxyUI.py	24 Sep 2003 05:28:53 -0000	1.26
--- ProxyUI.py	29 Sep 2003 07:26:28 -0000	1.27
***************
*** 560,564 ****
  
          self.write(box)
!         self._writePostamble()
  
      def _contains(self, a, b, ignore_case=False):
--- 560,564 ----
  
          self.write(box)
!         self._writePostamble(help_topic="review")
  
      def _contains(self, a, b, ignore_case=False):

Index: UserInterface.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/UserInterface.py,v
retrieving revision 1.26
retrieving revision 1.27
diff -C2 -d -r1.26 -r1.27
*** UserInterface.py	24 Sep 2003 05:28:53 -0000	1.26
--- UserInterface.py	29 Sep 2003 07:26:28 -0000	1.27
***************
*** 22,25 ****
--- 22,26 ----
    onConfig - present the appropriate configuration page
    onAdvancedconfig - present the appropriate advanced configuration page
+   onHelp - present the help page
  
  To Do:
***************
*** 83,87 ****
  from Options import options, optionsPathname, defaults, OptionsClass
  
! IMAGES = ('helmet', 'status', 'config',
            'message', 'train', 'classify', 'query')
  
--- 84,88 ----
  from Options import options, optionsPathname, defaults, OptionsClass
  
! IMAGES = ('helmet', 'status', 'config', 'help',
            'message', 'train', 'classify', 'query')
  
***************
*** 133,137 ****
          return False
  
!     def _getHTMLClone(self):
          """Gets a clone of the HTML, with the footer timestamped, and
          version information added, ready to be modified and sent to the
--- 134,138 ----
          return False
  
!     def _getHTMLClone(self, help_topic=None):
          """Gets a clone of the HTML, with the footer timestamped, and
          version information added, ready to be modified and sent to the
***************
*** 141,144 ****
--- 142,147 ----
          clone.footer.timestamp = timestamp
          clone.footer.version = Version.get_version_string(self.app_for_version)
+         if help_topic:
+             clone.helplink.href = "help?topic=%s" % (help_topic,)
          return clone
  
***************
*** 178,184 ****
          self.write(re.sub(r'\s*\s*', '', str(html)))
  
!     def _writePostamble(self):
          """Writes the end of time-consuming pages - see `_writePreamble`."""
!         self.write("" + self._getHTMLClone().footer)
          self.write("")
  
--- 181,187 ----
          self.write(re.sub(r'\s*\s*', '', str(html)))
  
!     def _writePostamble(self, help_topic=None):
          """Writes the end of time-consuming pages - see `_writePreamble`."""
!         self.write("" + self._getHTMLClone(help_topic).footer)
          self.write("")
  
***************
*** 887,888 ****
--- 890,911 ----
  
          options.update_file(optionsPathname)
+ 
+     def onHelp(self, topic=None):
+         self._writePreamble("Help")
+         helppage = self.html.helppage.clone()
+         if topic:
+             # Present help specific to a certain page.  We probably want to
+             # load this from a file, rather than fill up UserInterface.py,
+             # but for demonstration purposes, do this for now.
+             if topic == "review":
+                 helppage.helpheader = "Review Page Help"
+                 helppage.helptext = "This page lists messages that have " \
+                                     "arrived in the last %s days and that " \
+                                     "have not yet been trained.  You should " \
+                                     "go through the messages, correcting " \
+                                     "the classification where necessary, " \
+                                     "and then click train, to train " \
+                                     "SpamBayes with these messages" \
+                                     % (options["Storage", "cache_expiry_days"],)
+         self.write(helppage)
+         self._writePostamble()



From anadelonbrin at users.sourceforge.net  Mon Sep 29 03:26:31 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Mon Sep 29 03:26:38 2003
Subject: [Spambayes-checkins] spambayes/spambayes/resources ui.html, 1.22,
	1.23 ui_html.py, 1.22, 1.23
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes/resources
In directory sc8-pr-cvs1:/tmp/cvs-serv32479/spambayes/resources

Modified Files:
	ui.html ui_html.py 
Log Message:
Add a basic 'help' link to the web interface, which can be
a standard set of information, or specific to the page on
which the user clicked 'help'.  The actual help information
still needs to be added!

Index: ui.html
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/resources/ui.html,v
retrieving revision 1.22
retrieving revision 1.23
diff -C2 -d -r1.22 -r1.23
*** ui.html	29 Sep 2003 04:43:09 -0000	1.22
--- ui.html	29 Sep 2003 07:26:28 -0000	1.23
***************
*** 111,114 ****
--- 111,129 ----
      
  
+ 
                
+ 
                The help page
                
+ 
+ 
                
+     
+     
+     
+         
                SpamBayes Help
                
+              
+         
+         If you have fully exhausted all your ideas, then you can
+         contact
+         the SpamBayes team.
+     
                
+ 
                
  
  
                
***************
*** 524,527 ****
--- 539,546 ----
          Mon Dec 30 14:04:32 2002.
          Spambayes.org
+     
+         
+             Help
+         
      
          \
! GrU$>Fw\006.]L~~Ů&o߰_^y}px9T/N\006ވM2\
! Q\021Ʉ᫟\035o\0262߂X(\\|i\0144MeQe)\
! vK\014\021y\002\035\037?i]ky^,EPUB[ZL\001Q\\\004Cf\0256s\
! l\025\002\023#r\014d:f\\x=\032٪\0313x_li\003I/}Г\016\016\003֡4^\013\006\
! ;-\015V7Mn\003E&b(GEІrEet\005wg8Gޓӊ\017D_mgC\025gC\
! pIJD\013\0051sWy7y\012\010=\000\033\000\034,ZY\005z2G3\001`͓sOe\
! \024pd\007PhP\003Y0\0035\007K3\031A3_6\020\000OV8KSE&f0\020ܻ\
! \025<\033\022\0328ee4ez\006\0244C\022K\036%\027ʝ\035-[lqFGaZN\
! \011H\",\001\002\031\012\016X(fQ\002J(\026R~8Ix8\023\"V$ޔ\010FFGi[\001\012\017\030\
! {Y@\032\015؃d5\\\010kB&`\020L<\024\031\017a`\000$J2P$2)% d#(\024V\
! 8~dj6\005qB\005/|9㯜);VZ-\014\006\001\003bQ\024°\006r\004ԝpJ׏oE\
! \034\001JdR\022A`E6(4\034\030\026\001\017\037\000= i\"\036g,\007\017dS2#@ہ\012\
! rJ+\021\"Q7Sԧ@\022m\010g\024\032\013>? }O\005\010b\001 \003Z냳aj\033\013\012\
! ƗiL,J\007ZaV#*|\025X(\002\013P4M_i\037<,\035&\016+\036S\001\
! \032؏\003\026\027z0ĵsg9CZ1=RVmB0m/PF6nH^I\007x]{_\005q\
! )4[\010^}\010\012\035B\013\020߀́]f%؎9H\034\015\006\001sه\
! {;R\012e\000j\006\006\010TW\"CI3\025$6gL4.NM0D=\033F\"Y\\\014\032\
! \\,x2\027(a-\020쬳~h_dڵѧ\010'\010\026[ЎV\016[-4H\012\032|Y=(,EY\007/x+?M$\005S\001a8\036x\022I\035}\
! -U\0034Tw\033+\"\004{\011K]X1(E\006\006\004\017\025-ڧٰ\035`j\032fl\023\
! u^N\003\020j7:7\0157|Wu%dQhqK\022D\022ou\
! ;%thi}\015=ЖL6P,U\024*\002\"485^Q\013-x\002-\
! V@T\012-\035y\032KYuzo\032\005w\033=Q\011QV\024\"\\v\0345)5\006dp<\
! k/I^fխm!'\017&daԒ\0049\037@Λ}]\026\036mA Ÿtu\
! xp5h?[}l\034J\001`@h\011M\000FU\0252#3|\011լc篐+\004X`Є\004f\017Yd\
! \0131/\022\020^\007*-cĠFt\013[\004а0̌\"ǩ\023\015tZ\013X\"\0370\025\022\
! —&1\0039\027\000G\005ӭ2Zt\035Ru)P\011\004Ȋح_\032'\"e*|U\
! \000ո\000\020֪tr8jX\012\004 u\016qu^P=IQ17jHY-;rZPTZ\
! 0\032V\016kA⫢F\023\037C4\003p2& \016\007O&\026zb\011%\"\012C\010\030\026!].\011W_l\
! @ \001RUow\025@\0145\002/s\035XA^NQa8K4[n_\037iQ_Cf\020F9\
! \007$FV-\021\000*8܅\013m@6\016nt/\036\017?CO\025xZu==e#ү!)ȢԈ\005\
! ?=WO]8<\027\027@(\002=I;EH]TX\023^(r\011\000ӧ\
! d\020d\016yΎ\037Z$/;oumi\031ۋw\"vF\000/o6؝\034'ڄ\000\"\
! \001^cwg#\013\005z\0321[\000~!j\015\031S+\016\012FDv\001~\
! \004\011U\001a\024=2Ⱥ՗puHƲ#\032^~;KtMB%>\000Umɨ\021o\
! ˩YL\021PA\0205\013хxIʬL)\0065p)n`\020\004llH*P7WݱvЩ\
! sN\0304uQ\020\015A5M9v\033_Z9\035\004\0022nVlD\033\
! =+Ze'r\024\026\011GN\015\032\\\013\000\"#\025\000\0148Xqy(\"Zx\005 \":\013\
! `,Fd\036J[]\001\011 :@\014nj'\033h/nr\032$n.\004}o\0359\
! f\0121p]7Q\004\020@\022w%2zA``1\011\032|\014=\023e\"6\
! i9\003\022h\023DM\032\001:qJpoSLL\006\037cFZ\012yqF#v+Rvyl\
! ;\036F=Ub]\010\012r'\000x\030I\023\007EqL?\014DB\032fU\010G0p\
! Q),`I\016ep\026\003x,To%(xs\006\015\023zBVAb|@Bm\014lR\007J;8YJיfʥk\025zݚ\
! Y-duҢO(X[\001@wN\025\032׈|[q\021<+Ð\027ܬ\014\"\004U\0317e\
! \013kqC\0370{AD\033ͣ\032jD|n:Alh7YPMeqU\0074+\011c\
! \012Oe\013\004\004#ђ\007ʱIUs\027e\022e\001wcb\005i\024\030.ݥ\
! DrW\033\036n[3\026Y\004B'llcJ,ob;ўx\022D6\016u%\
! zۑeb)0WuhBJf\"{T=P\001\037\023\037i8Ճv'B;\014Ϟ=iFTS\
! \034i\015ڪo\030{5\001\007=\022 \004\027'-ѪК\032{͹'\001f3$\027\010y\036\
! \005F\032R\035r6;\003\013uW:6\021tB\012Ts0XLD.L+\000t%\036\021\
! TĎ\022oPt-,FqbZ\024Ӥ.&\031ͨF%\\\003\"u\001TS\003#H8UlՎZچLyo,\
! \025v\014xo\013^kI?)nrF'\015\036|\027\022\031\031\035\0359PUZ\010<{u\
! &|\032Q댒dk\030$2hR6UL\035\033\037\0141\020\036f\015mY\006q@B\
! \034U\\g\010\016d\010\0160Yd2%\026MO1ZGt9LU¿\027ӰfRO\006H\
! \010/1~^\035VNxw鳨\034?}\03045W_^1i>n5N񕴞\
! wsg#]wʑ%$q/]R.\025gی~RֹqGe\005l'Ԃ\
! >_|P\0000EN\000!Sw\022֍U~u\030?\001Q<{62֘Fr\015_\
! jT'\021~\005\032[\002F<]۲^\005S1@ +؊iw](h펣m\
! Th@\033QYR\037\010l~;T'tlj\013Q;S\005)\003\
! mRK߻N\014\005]T{˛ٺ~\\;+5&\024GX.qHs\030JMb\037oRUkV\
! x\016\005b\012]ILp\001׵k.mYS!3kMEeCH\036\014F\
! y-\"Z/5)\036~G\007nC3Vzd\014Xߊjt0܀M\027S^\
! u1z\014:,̵=Xͧ6\025CZ[A⻊U>Rfw?{ls\
! Wk\013.GTt]\033R&T!f\\U\021\026Nu<ިa\032߭ܚmFY,;\
! f#N}G&\027>k|(ӟR'[(P\035\007UQ:'Q~2>)C\
! \017#|\017\0052[\0279\030O\000}\015zޓ݌޿Z-ESwuDAi\032\
! a\027\035\007r\025e\037'և~\026\0205;[\030R\004cw2>>£sGA\007=fQ\
! pZ\015\002@f\010޷'ՙ^\032JjL1݈Oζj0Ut/IF)\036LSf:\
! Y\030ŭN(DrD3\031>r\002C\021o")
  ### end
--- 7,104 ----
  import zlib
  data = zlib.decompress("x\\{s6?U\016\010S6Zi#d$ΖX'M֕JT\030\02235η\033\000\011>\
! GrU'!\0014\032\037\033}wb+v5y]\013\034\016/ɥ*8\031x#6x\
! GE$\023\036\017~vo9[\024˘\012\036\"*bܡO?N\012x ΆT\001,E\
! ٢(C2?w.x\020\016\013$TNs'z5!2[-&\001\0236\0361g?\
! #C\036^\000L=\037e\027q\024\002^@Q^p\0274\014\037G6\003\012c,\001WQ\
! \037E\021\005-y61ؿS\020\016LfK 7|\034G\007x\037Xfc\007?\001\025}\004\
! '5\013y\017\023T=N\014\006Z\024k\016W\"/ǩCq\037\025ܩ\0003Y\
! &!\024x\002z:e)\017(?>\036b\0064FNU?S\";,d\012E\003%\007@T\026\\v\022\
! \020G-ưk)QR9\0135_\005\012ůx)9;j\000m̵f\037DQrHbɣBGhVr>[\021q`0-'\
! \004d\021l\026\000g\014_qAO\007,\024(\001!\024\013b)?\034$<\011\021+\022rF%%.y[Vd\003\
! &\036_\"F9%Yu\027š\011X&\004\023\017E\003Tp\030\000+Ҹ9(\024Lh\011%Y\010\002\
! $\037MA\020b&_b'ge\016\026\013d?`@,B\030VU@.Ss\030[\021\
! o(\020Yķ\004~\020X\015`>\012\015\007EC9\007@\0174M\021\017c3Sm\003\007cS2#@ہ\012\
! r%J*\021\"Q6SħA%Z\021(4\014\027|~@\012\030 vʮ\037\005\016j\017ΆP/*\
! \032_1q+Jf\037hYT0*\001Vb\0106/@B4.6x\026%iY9U\034V<\002\
! 5\037\0074..EU0ĵsg9CmZ1=\022VB0m/PJ6n\036\017=_\005q\
! )T[*\017^}\010\012\035B\013l%sYa\015lvhT${\006\0039YB\
! ý\035)2\000B5\003\003\004*+!]ɤ\031\012\022\033Գk&\032\027_M&E\030\"\015#!Y\\\014\032\
! ѹXd.BpHgC$֦6E8QFW*߂v\\r]䉫\027\021\006\0338|*\
! \005\027z\037\034\007J\003&3\001\022\032A\025Y&,\0172^\000E-\027I]\020G\002\025\003\030\000\
! >\032Tb !4?d\001\011;GbdB\025\030\007c\031\014W\015\006\020N%\016\034PL\021\0001\021\032n[B:$\
! V`.*nP\022j\004\037RĀXE|ga\0055$Љ\025\027\007t\0052R(\036\
! c\024Dtx\0341]\030mRvlech\031`֘Ud\031x\034:փ]GCQ׌\
! \010N\033\"\\@.Z\004\020H-耺*\002\001Ҟ\006*\003/\020s\027(so'\013\030z\
! 8|Zz0jAn\031.\022Wn<)О\016\030\016݁\024NjWn\003\
! ܬm\012\036Z^_\021!KX2a}ā\031_d` L B]c^4\033\035߭zÀ %d\
! 2[9X\005_\025pYC?%Wv~]u\"!lV3\026^YxX2\
! '\017\035P\014\012AMC\003ԩj\007\001\024L\016fCy\017d6\037\"\023\034\021_?Ս?\\\010pk\
! YQfR@C<(\035U8gG\035\035Xm\0352&\003ͩ\012\025eg]x)\0335]\0373*D\
! <\005\012Ll귚^KҚfv6\0076j@]Z#e\010HYEA;'\003X,B\003ڠ\005(\020\
! h\011q\013\023\010$qK>95\025\020@MǷy\036:RVN@Zoꦐ\005w\021S\011QZ\024\"v\
! 5)\016\031ZfWV~I2\023f%l\017r`B\014|#[#Aǭ>&fnE\
! &q[`@\003(R|.\036\\\015oV\033\033on\014\023(,o\002Qxl;5UEH\015`^Ȫ\
! W\005\020\022\012IEKJ\037\0018h,\032]\036^^yü͜m,ozz:~\025KT\
! \013\023O[,!|i^\0363\034\033\002pT0]5(N\"f\
! H\030!聢c\004UZtb\004GR4`ZJ{\0012S\015XժQz\020\007\032\032OA\020=cNf\
! Z\026\005b\006\001\027\021?i;A.bh\005K1\023@uZS\013F5E92\015{(rL\003\021/D\
! U(\017x\026!#\027p\016~\0220\007\001\015Uk5\\9\034\034-eO*05,W\035t\
! k\017C%(d\034}\010;F3\0368NT\004ABO\020\011s\027Kf\
! \033*cz6rn eSosiG\014\032y:/ /˨0\034̥K-o'\0114w(\
! /Y,Q\001Ep\000\025\034\0056 b\033\0077\027Rm?CO\035𺵪\035tcF̦eF\
! \000YCRE\031\026F1{޺pxo\027/&/~92Q&'\023\005v{ؒ6`1:W'P\006\
! \022\001򑳧OY \026ɼX0\035M?6HVvjN5\027AEE쌚\016_߬\
! ;U9\004崷\005Do\003\001\025cwg\024\007Z4#:?g8ު\000ŗ(5x߯&O5=;(\
! \033\022\005o{\0377\023$T\005R_W֣\014^\037HYvDw\023oGI,\000\
! ^\025D񶜚ո$\033\016\0258Q[\000X\021\011\031\031o̪T\035Q\035$D\033is+P\
! \017WMN:V#tƪ\012 \032ܚ\034jv\033_Z1\004\002\"nѐ\033.\
! 熚\023u\037p+WkUS[$;5\010ksQ/\003\027\00303ϣF~\026O\
! _@\011|)sr;I\005}1\"\037aω\031\000\011D\026?\003ELUč\\I;s\
! C˶М\002Ȩgc\025bro`\021\011v\025%!$nvea\003>d\022nap\030\
! z&f\027\017cbڄ5\017H\016̧4\002u\021c+-s2\031|\037\016\012\0321UX\004j\
! \0036\032[#;f\037x4tEd;\016xH\032?P/\"cZ?կn \027*0\
! ~\\?\011\033Je\001SZc\014.\003\030f?i+7'=M㨿\
! 0j\014z/\020_\010s\037DwL}\013\031[&]y\001ϯ\012\001K\035(`d,]\
! g!UISOkf7\011m2%\017\031\000*s*$P'\004S4e\031\027Qʳ:\
! \014y*uEq\026n\007\037\003@8}C\011 #7T8Al萂\
! j!*\017\004aV\022%m\025|Bc{\022\027\010\011{%\017@c&.5ޭ3KJ\003\
! )j\001\013dS)ЛMKo6R-z;\013]&g4J\011aTO<\034\017]8w*P\003rFG\
! -\0042)uhN\011ֻ,\023KyOg\004j;':W2\013\024٣j4\006|O|\005T/\
! @\013aC0<{\033QM\001\022ssV~Uc\011\030>1\000'>cN,So\
! <\0110i\006G (0!\021!7ns\021=J&.X1eHa7yj\016\006Kp\034\
! (\037ԉ)u:\000]\015|*b\017LE+\026i\0268TQwz(JS\030ݑX7j^?\
! H31\027\017\033I\030K3\0331/cW\017x\005wh?{ࣿ\024\\\001[\0217ީqbâgj\
! eXM\030&v涢g{\037edu?z\026\0053\\U5ܑ@L$AՉ&qyH\015\017h]`\
! \016m\031\004:o\002Y\025yT慻\035u*͑g3\001SV\016D\016\006V\0276\
! TR\024:mlFv\003\0036/A\006FrsS\017&[X\017PU\023s\016c?^3\
! Q$Vԙ6n\0144\000w8\034ղ55ᬃo;1\026ԗg\000U\007u΁jJc\004\011\012\
! >;QW)Ŷ\002ߎ\016m_*6sB>Rpn@x\025itt@\
! UMjҢ\"t\032Bיi\024G׮#J:\"ʯӵȠ\011!یCT1u|o\037~\\a\0353\
! =x\033Ҳ\015Rq@B\034䪓W#\004\007\"\004\007,2\022'\031Q#O:\034C{2_H\
! iX3G'Ag$in}2\030:U5uC*ufQ599~{7j4m\027=c\033\003\
! }ݪdܡ+i=;cXFH@O;\023!\020ǽzwIT\004)k3]K\
! [\021\012\000O\006K}l\007\000`p\000BT\013\022֕U~u\030r\001(\
! C=\033\031kf#-}5\010/\001\026h@8O׶Wq6\014ᕾ\"a彭\
! 4x\027\0128kh[#k:.5РF\024e\007\002W1jl\016MMUz8*}g|\
! :gl.`EM\006TU=g2\034%d>\034F;vT[ю\007\"t\\\033\
! tb\017\023w9Q\013s\014oKy\000\034\017\003t. \031*Rw7n>z_b\
! !>r?LoEl\027p2y+Nm\013\033\025x`2\025:u\011\
! ?}φ\025M\036=!\015k\027w)ϖw\035-y4s\0158\016M\
! g\022蹩\007[kP7\030\01665\027>SjY1q&=tю]Pj\022\
! \000՞~\023Z\023G:\024*t\007\015M`J\013=}]W\002:j5\037RD_Y\
! IC\034\014͐rd\"RC\"O|W}8\032!A[Pҕ3\031\003ַ8\
! #97 l\024kMշ\0223E\021+&\030yubHkk\011RWɿX\
! #vw*c'%\005FZ^8u:ۿp2\0161*KpoFUċ\021\036~\
! rk\031|jf1;ub\013M~5/4}ָ(7lcS}|Q\026\\%};\016VQ:'ߑ\
! ~2>_ozB\002E\034{YǾ\006b=InFW8VKg]}`\
! xȠ4c\015\003_\014VF{}պ؉g\001^㿅/E=6:\031{'#<:w\031\031t\
! $Jj\0324%[sxRG,.\034\001\030n@\00383s4L\020l \014\
! c(\005\016\010ζt6t+}UwdnSSf\032a\036ŭ\013\025Q}\033\
! F\005x")
  ### end



From anadelonbrin at users.sourceforge.net  Mon Sep 29 03:28:14 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Mon Sep 29 03:28:17 2003
Subject: [Spambayes-checkins] spambayes/spambayes/resources help.gif, NONE,
	1.1 help_gif.py, NONE, 1.1 .cvsignore, 1.1, 1.2
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes/resources
In directory sc8-pr-cvs1:/tmp/cvs-serv32763/spambayes/resources

Modified Files:
	.cvsignore 
Added Files:
	help.gif help_gif.py 
Log Message:
A not-quite-matching help image for the web interface.
If whoever made all the others is reading this, it would be
great if you could replace this with a matching one :)

--- NEW FILE: help.gif ---
(This appears to be a binary file; contents omitted.)

--- NEW FILE: help_gif.py ---
# -*- coding: ISO-8859-1 -*-
"""Resource help_gif (from file help.gif)"""
# written by resourcepackage: (1, 0, 0)
source = 'help.gif'
package = 'spambayes.resources'
data = "GIF89aq\000q\000\000\0003f@p9k!\025Created with The GIMP\000!\004\001\012\000\003\000,\000\
\000\000\000q\000q\000\000\002m\000ًδ{h}9z\0132\013o0XP5\013\"x(K \014\026\
\023fz.Rߧڈb[׭CUt%\020@5\0367Z;ȭ#\037UDr\007($8H\027T(\
H\002\007Q7#\011\010BA\011嘙cc\031I\030\030\012::W\001\021j\033[\0120\"\031+\
q\034<%Q\004Sd\015=Hː,\016\011NNuXp.t+.bB=/~\
A4^=f\\\013\010=c`%x\030o\014\027}o2\003CrȎ\000OK\030\0060\024\017\
Ke\023\020JTXFgI8Er(M(d\032nFM\024\026̅\0073a~\"[ꙉ\006\016*\
Mhj侥[Wm[w+\036\004\027^\0157v\034ϵ#G4WVjN=4+h~\
t&gd\001H]:/ӱ\012͏u۷۸\013\015qq\012l\015\036w*uvG\023}Ss\
v\003+\\e=m^w7vy\007α0\016,gz\036\036e%Q\037z-܀\000*\
i`J3\005H'\022>E\035t\010\035\033N˃\027á\007\"\037'ڒKV\"R\032j\025c fy\
A\010ފwWBx\011

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv16023/spambayes

Modified Files:
	FileCorpus.py ProxyUI.py 
Log Message:
Fix [ 814322 ] AttributeError: hdrtxt

Basically the review page would die if one of the cached messages was moved from
the cache directory by someone other than spambayes (for example a virus
protection program).  Instead, we ignore the problem and just don't present that message
for review.

Note that you can actually still train some messages that have been removed,
as long as they are in the memory cache.  I can't think of any problems that this
would cause.

Index: FileCorpus.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/FileCorpus.py,v
retrieving revision 1.7
retrieving revision 1.8
diff -C2 -d -r1.7 -r1.8
*** FileCorpus.py	19 Sep 2003 23:38:10 -0000	1.7
--- FileCorpus.py	30 Sep 2003 03:05:13 -0000	1.8
***************
*** 219,245 ****
          pn = self.pathname()
  
          try:
!             fp = gzip.open(pn, 'rb')
          except IOError, e:
!             if e.errno != errno.ENOENT:
!                 raise
!         else:
!             try:
                  self.setPayload(fp.read())
-             except IOError, e:
-                 if str(e) == 'Not a gzipped file':
-                     # We've probably got both gzipped messages and
-                     # non-gzipped messages, and need to work with both.
-                     fp.close()
-                     try:
-                         fp = open(self.pathname(), 'rb')
-                     except IOError, e:
-                         if e.errno != errno.ENOENT:
-                             raise
-                     else:
-                         self.setPayload(fp.read())
-                         fp.close()
-             else:
                  fp.close()
          self.loaded = True
  
--- 219,235 ----
          pn = self.pathname()
  
+         fp = gzip.open(pn, 'rb')
          try:
!             self.setPayload(fp.read())
          except IOError, e:
!             if str(e) == 'Not a gzipped file':
!                 # We've probably got both gzipped messages and
!                 # non-gzipped messages, and need to work with both.
!                 fp.close()
!                 fp = open(self.pathname(), 'rb')
                  self.setPayload(fp.read())
                  fp.close()
+         else:
+             fp.close()
          self.loaded = True
  
***************
*** 260,268 ****
      def remove(self):
          '''Message hara-kiri'''
- 
          if options["globals", "verbose"]:
              print 'physically deleting file',self.pathname()
! 
!         os.unlink(self.pathname())
  
      def name(self):
--- 250,262 ----
      def remove(self):
          '''Message hara-kiri'''
          if options["globals", "verbose"]:
              print 'physically deleting file',self.pathname()
!         try:
!             os.unlink(self.pathname())
!         except OSError:
!             # The file probably isn't there anymore.  Maybe a virus
!             # protection program got there first?
!             if options["globals", "verbose"]:
!                 print 'file', self.pathname(), 'can not be deleted'
  
      def name(self):

Index: ProxyUI.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/ProxyUI.py,v
retrieving revision 1.27
retrieving revision 1.28
diff -C2 -d -r1.27 -r1.28
*** ProxyUI.py	29 Sep 2003 07:26:28 -0000	1.27
--- ProxyUI.py	30 Sep 2003 03:05:14 -0000	1.28
***************
*** 489,492 ****
--- 489,493 ----
                              options["Headers", "header_spam_string"]: [],
                              }
+         invalid_keys = []
          for key in keys:
              if isinstance(key, types.TupleType):
***************
*** 497,501 ****
              # info object for each message.
              message = sourceCorpus[key]
!             message.load()
              judgement = message[options["Headers",
                                          "classification_header_name"]]
--- 498,509 ----
              # info object for each message.
              message = sourceCorpus[key]
!             try:
!                 message.load()
!             except IOError:
!                 # Someone has taken this file away from us.  It was
!                 # probably a virus protection program, so that's ok.
!                 # Don't list it in the review, though.
!                 invalid_keys.append(key)
!                 continue
              judgement = message[options["Headers",
                                          "classification_header_name"]]
***************
*** 506,509 ****
--- 514,519 ----
              messageInfo = self._makeMessageInfo(message)
              keyedMessageInfo[judgement].append((key, messageInfo))
+         for key in invalid_keys:
+             keys.remove(key)
  
          # Present the list of messages in their groups in reverse order of



From anadelonbrin at users.sourceforge.net  Mon Sep 29 23:16:14 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Mon Sep 29 23:16:16 2003
Subject: [Spambayes-checkins] spambayes/spambayes Corpus.py, 1.7,
	1.7.2.1 FileCorpus.py, 1.6, 1.6.2.1 ProxyUI.py, 1.23.2.1, 1.23.2.2
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv17907/spambayes

Modified Files:
      Tag: release_1_0
	Corpus.py FileCorpus.py ProxyUI.py 
Log Message:
Fix [ 814322 ] AttributeError: hdrtxt

Basically the review page would die if one of the cached messages was moved from
the cache directory by someone other than spambayes (for example a virus
protection program).  Instead, we ignore the problem and just don't present that message
for review.

Note that you can actually still train some messages that have been removed,
as long as they are in the memory cache.  I can't think of any problems that this
would cause.

Index: Corpus.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Corpus.py,v
retrieving revision 1.7
retrieving revision 1.7.2.1
diff -C2 -d -r1.7 -r1.7.2.1
*** Corpus.py	4 May 2003 03:05:33 -0000	1.7
--- Corpus.py	30 Sep 2003 03:16:12 -0000	1.7.2.1
***************
*** 288,292 ****
      def __getattr__(self, attributeName):
          '''On-demand loading of the message text.'''
- 
          if attributeName in ('hdrtxt', 'payload'):
              self.load()
--- 288,291 ----
***************
*** 346,350 ****
      def getSubstance(self):
          '''Return this message substance'''
- 
          return self.hdrtxt + self.payload
  
--- 345,348 ----

Index: FileCorpus.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/FileCorpus.py,v
retrieving revision 1.6
retrieving revision 1.6.2.1
diff -C2 -d -r1.6 -r1.6.2.1
*** FileCorpus.py	16 Sep 2003 04:42:32 -0000	1.6
--- FileCorpus.py	30 Sep 2003 03:16:12 -0000	1.6.2.1
***************
*** 220,246 ****
          pn = self.pathname()
  
          try:
!             fp = gzip.open(pn, 'rb')
          except IOError, e:
!             if e.errno != errno.ENOENT:
!                 raise
!         else:
!             try:
                  self.setSubstance(fp.read())
-             except IOError, e:
-                 if str(e) == 'Not a gzipped file':
-                     # We've probably got both gzipped messages and
-                     # non-gzipped messages, and need to work with both.
-                     fp.close()
-                     try:
-                         fp = open(self.pathname(), 'rb')
-                     except IOError, e:
-                         if e.errno != errno.ENOENT:
-                             raise
-                     else:
-                         self.setSubstance(fp.read())
-                         fp.close()
-             else:
                  fp.close()
  
      def store(self):
--- 220,236 ----
          pn = self.pathname()
  
+         fp = gzip.open(pn, 'rb')
          try:
!             self.setSubstance(fp.read())
          except IOError, e:
!             if str(e) == 'Not a gzipped file':
!                 # We've probably got both gzipped messages and
!                 # non-gzipped messages, and need to work with both.
!                 fp.close()
!                 fp = open(self.pathname(), 'rb')
                  self.setSubstance(fp.read())
                  fp.close()
+         else:
+             fp.close()
  
      def store(self):
***************
*** 257,265 ****
      def remove(self):
          '''Message hara-kiri'''
- 
          if options["globals", "verbose"]:
              print 'physically deleting file',self.pathname()
! 
!         os.unlink(self.pathname())
  
      def name(self):
--- 247,259 ----
      def remove(self):
          '''Message hara-kiri'''
          if options["globals", "verbose"]:
              print 'physically deleting file',self.pathname()
!         try:
!             os.unlink(self.pathname())
!         except OSError:
!             # The file probably isn't there anymore.  Maybe a virus
!             # protection program got there first?
!             if options["globals", "verbose"]:
!                 print 'file', self.pathname(), 'can not be deleted'
  
      def name(self):

Index: ProxyUI.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/ProxyUI.py,v
retrieving revision 1.23.2.1
retrieving revision 1.23.2.2
diff -C2 -d -r1.23.2.1 -r1.23.2.2
*** ProxyUI.py	24 Sep 2003 03:54:15 -0000	1.23.2.1
--- ProxyUI.py	30 Sep 2003 03:16:12 -0000	1.23.2.2
***************
*** 390,398 ****
                              options["Headers", "header_spam_string"]: [],
                              }
          for key in keys:
              # Parse the message, get the judgement header and build a message
              # info object for each message.
              cachedMessage = sourceCorpus[key]
!             message = spambayes.mboxutils.get_message(cachedMessage.getSubstance())
              judgement = message[options["Headers",
                                          "classification_header_name"]]
--- 390,407 ----
                              options["Headers", "header_spam_string"]: [],
                              }
+         invalid_keys = []
          for key in keys:
              # Parse the message, get the judgement header and build a message
              # info object for each message.
              cachedMessage = sourceCorpus[key]
!             try:
!                 message = spambayes.mboxutils.get_message(\
!                   cachedMessage.getSubstance())
!             except IOError:
!                 # Someone has taken this file away from us.  It was
!                 # probably a virus protection program, so that's ok.
!                 # Don't list it in the review, though.
!                 invalid_keys.append(key)
!                 continue
              judgement = message[options["Headers",
                                          "classification_header_name"]]
***************
*** 403,406 ****
--- 412,417 ----
              messageInfo = self._makeMessageInfo(message)
              keyedMessageInfo[judgement].append((key, messageInfo))
+         for key in invalid_keys:
+             keys.remove(key)
  
          # Present the list of messages in their groups in reverse order of



From anadelonbrin at users.sourceforge.net  Tue Sep 30 00:18:08 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Tue Sep 30 00:18:12 2003
Subject: [Spambayes-checkins] spambayes/spambayes Stats.py, NONE,
	1.1 UserInterface.py, 1.27, 1.28
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv27095/spambayes

Modified Files:
	UserInterface.py 
Added Files:
	Stats.py 
Log Message:
Add the ability to display basic statistics.  At the moment this is just what we
can generate with no extra effort from the messageinfo database (as long
as it hasn't corrupted yet ).  This is deliberately designed to be very
similar to the oastats.py Stats class - maybe one day they'll be the same,
although it wouldn't make much sense at the moment.

--- NEW FILE: Stats.py ---
#! /usr/bin/env python

"""message.py - Core Spambayes classes.

Classes:
    Stats - provides statistical information about previous activity.

Abstract:

    Provide statistics on the activity that spambayes has done - for
    example the number of messages classified as each type, and the
    number of messages trained as each type.  This information is
    retrieved from the messageinfo database, so is as reliable as that
    is .

To Do:
    o People would like pretty graphs, so maybe that could be done.
    o People have requested time-based statistics - mail per hour,
      spam per hour, and so on.
    o The possible stats to show are pretty much endless.  Some to
      consider would be: percentage of mail that is fp/fn/unsure,
      percentage of mail correctly classified.
    o Suggestions?

"""

# This module is part of the spambayes project, which is Copyright 2002-3
# The Python Software Foundation and is covered by the Python Software
# Foundation license.

__author__ = "Tony Meyer "
__credits__ = "Mark Hammond, all the spambayes folk."

from spambayes.message import msginfoDB

class Stats(object):
    class __empty_msg:
        def getId(self):
            return self.id

    def __init__(self):
        self.CalculateStats()

    def Reset(self):
        self.cls_spam = 0
        self.cls_ham = 0
        self.cls_unsure = 0
        self.trn_spam = 0
        self.trn_ham = 0
        self.trn_unsure_ham = 0
        self.trn_unsure_spam = 0
        self.fp = 0
        self.fn = 0
        self.total = 0

    def CalculateStats(self):
        self.Reset()
        for msg in msginfoDB.db:
            self.total += 1
            m = self.__empty_msg()
            m.id = msg
            msginfoDB._getState(m)
            if m.c == 's':
                self.cls_spam += 1
                if m.t == 0:
                    self.fn += 1
            elif m.c == 'h':
                self.cls_ham += 1
                if m.t == 1:
                    self.fp += 1
            elif m.c == 'u':
                self.cls_unsure += 1
                if m.t == 0:
                    self.trn_unsure_ham += 1
                elif m.t == 1:
                    self.trn_unsure_spam += 1
            if m.t == 1:
                self.trn_spam += 1
            elif m.t == 0:
                self.trn_ham += 1

    def GetStats(self):
        if self.total == 0:
            return ["SpamBayes has processed zero messages"]
        chunks = []
        push = chunks.append
        perc_ham = 100.0 * self.cls_ham / self.total
        perc_spam = 100.0 * self.cls_spam / self.total
        perc_unsure = 100.0 * self.cls_unsure / self.total
        format_dict = dict(perc_spam=perc_spam, perc_ham=perc_ham,
                           perc_unsure=perc_unsure, num_seen = self.total)
        format_dict.update(self.__dict__)
        push("SpamBayes has processed %(num_seen)d messages - " \
             "%(cls_ham)d (%(perc_ham)d%%) good, " \
             "%(cls_spam)d (%(perc_spam)d%%) spam " \
             "and %(cls_unsure)d (%(perc_unsure)d%%) unsure." % format_dict)
        push("%(trn_ham)d message(s) were manually " \
             "classified as good (with %(fp)d " \
             "being false positives)." % format_dict)
        push("%(trn_spam)d message(s) were manually " \
             "classified as spam (with %(fn)d " \
             "being false negatives)." % format_dict)
        push("%(trn_unsure_ham)d unsure message(s) were manually " \
             "identified as good, and %(trn_unsure_spam)d as spam." \
             % format_dict)
        return chunks

if __name__=='__main__':
    s = Stats()
    print "\n".join(s.GetStats())

Index: UserInterface.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/UserInterface.py,v
retrieving revision 1.27
retrieving revision 1.28
diff -C2 -d -r1.27 -r1.28
*** UserInterface.py	29 Sep 2003 07:26:28 -0000	1.27
--- UserInterface.py	30 Sep 2003 04:18:05 -0000	1.28
***************
*** 82,85 ****
--- 82,86 ----
  import Dibbler
  import tokenizer
+ from spambayes import Stats
  from Options import options, optionsPathname, defaults, OptionsClass
  
***************
*** 892,895 ****
--- 893,898 ----
  
      def onHelp(self, topic=None):
+         """Provide a help page, either the default if topic is not
+         supplied, or specific to the topic given."""
          self._writePreamble("Help")
          helppage = self.html.helppage.clone()
***************
*** 898,901 ****
--- 901,905 ----
              # load this from a file, rather than fill up UserInterface.py,
              # but for demonstration purposes, do this for now.
+             # (Note that this, of course, should be in ProxyUI, not here.)
              if topic == "review":
                  helppage.helpheader = "Review Page Help"
***************
*** 909,911 ****
--- 913,927 ----
                                      % (options["Storage", "cache_expiry_days"],)
          self.write(helppage)
+         self._writePostamble()
+ 
+     def onStats(self):
+         """Provide statistics about previous SpamBayes activity."""
+         # Caching this information somewhere would be a good idea,
+         # rather than regenerating it every time.  If people complain
+         # about it being too slow, then do this!
+         s = Stats.Stats()
+         self._writePreamble("Statistics")
+         stats = s.GetStats()
+         stats = self._buildBox("Statistics", None, "

                ".join(stats))
+         self.write(stats)
          self._writePostamble()



From anadelonbrin at users.sourceforge.net  Tue Sep 30 00:18:08 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Tue Sep 30 00:18:16 2003
Subject: [Spambayes-checkins] spambayes/spambayes/resources ui.html, 1.23,
	1.24 ui_html.py, 1.23, 1.24
Message-ID: 

Update of /cvsroot/spambayes/spambayes/spambayes/resources
In directory sc8-pr-cvs1:/tmp/cvs-serv27095/spambayes/resources

Modified Files:
	ui.html ui_html.py 
Log Message:
Add the ability to display basic statistics.  At the moment this is just what we
can generate with no extra effort from the messageinfo database (as long
as it hasn't corrupted yet ).  This is deliberately designed to be very
similar to the oastats.py Stats class - maybe one day they'll be the same,
although it wouldn't make much sense at the moment.

Index: ui.html
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/resources/ui.html,v
retrieving revision 1.23
retrieving revision 1.24
diff -C2 -d -r1.23 -r1.24
*** ui.html	29 Sep 2003 07:26:28 -0000	1.23
--- ui.html	30 Sep 2003 04:18:06 -0000	1.24
***************
*** 134,144 ****
          
          
!         POP3 proxy running on 1110,
!         proxying to example.com.
!         
                
!         Active POP3 conversations:
!             0.
                
!         POP3 conversations this session:
!             0.
                
                          
          Emails classified this session:
--- 134,144 ----
          
          
!             POP3 proxy running on 1110,
!             proxying to example.com.
!             
                
!             Active POP3 conversations:
!                 0.
                
!             POP3 conversations this session:
!                 0.
                
                          
          Emails classified this session:
***************
*** 149,152 ****
--- 149,153 ----
              Spam: 0
              Ham: 0
                
+         More statistics...
                          
      

Index: ui_html.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/resources/ui_html.py,v
retrieving revision 1.23
retrieving revision 1.24
diff -C2 -d -r1.23 -r1.24
*** ui_html.py	29 Sep 2003 07:26:28 -0000	1.23
--- ui_html.py	30 Sep 2003 04:18:06 -0000	1.24
***************
*** 6,104 ****
  
  import zlib
! data = zlib.decompress("x\\{s6?U\016\010S6Zi#d$ΖX'M֕JT\030\02235η\033\000\011>\
! GrU'!\0014\032\037\033}wb+v5y]\013\034\016/ɥ*8\031x#6x\
! GE$\023\036\017~vo9[\024˘\012\036\"*bܡO?N\012x ΆT\001,E\
! ٢(C2?w.x\020\016\013$TNs'z5!2[-&\001\0236\0361g?\
! #C\036^\000L=\037e\027q\024\002^@Q^p\0274\014\037G6\003\012c,\001WQ\
! \037E\021\005-y61ؿS\020\016LfK 7|\034G\007x\037Xfc\007?\001\025}\004\
! '5\013y\017\023T=N\014\006Z\024k\016W\"/ǩCq\037\025ܩ\0003Y\
! &!\024x\002z:e)\017(?>\036b\0064FNU?S\";,d\012E\003%\007@T\026\\v\022\
! \020G-ưk)QR9\0135_\005\012ůx)9;j\000m̵f\037DQrHbɣBGhVr>[\021q`0-'\
! \004d\021l\026\000g\014_qAO\007,\024(\001!\024\013b)?\034$<\011\021+\022rF%%.y[Vd\003\
! &\036_\"F9%Yu\027š\011X&\004\023\017E\003Tp\030\000+Ҹ9(\024Lh\011%Y\010\002\
! $\037MA\020b&_b'ge\016\026\013d?`@,B\030VU@.Ss\030[\021\
! o(\020Yķ\004~\020X\015`>\012\015\007EC9\007@\0174M\021\017c3Sm\003\007cS2#@ہ\012\
! r%J*\021\"Q6SħA%Z\021(4\014\027|~@\012\030 vʮ\037\005\016j\017ΆP/*\
! \032_1q+Jf\037hYT0*\001Vb\0106/@B4.6x\026%iY9U\034V<\002\
! 5\037\0074..EU0ĵsg9CmZ1=\022VB0m/PJ6n\036\017=_\005q\
! )T[*\017^}\010\012\035B\013l%sYa\015lvhT${\006\0039YB\
! ý\035)2\000B5\003\003\004*+!]ɤ\031\012\022\033Գk&\032\027_M&E\030\"\015#!Y\\\014\032\
! ѹXd.BpHgC$֦6E8QFW*߂v\\r]䉫\027\021\006\0338|*\
! \005\027z\037\034\007J\003&3\001\022\032A\025Y&,\0172^\000E-\027I]\020G\002\025\003\030\000\
! >\032Tb !4?d\001\011;GbdB\025\030\007c\031\014W\015\006\020N%\016\034PL\021\0001\021\032n[B:$\
! V`.*nP\022j\004\037RĀXE|ga\0055$Љ\025\027\007t\0052R(\036\
! c\024Dtx\0341]\030mRvlech\031`֘Ud\031x\034:փ]GCQ׌\
! \010N\033\"\\@.Z\004\020H-耺*\002\001Ҟ\006*\003/\020s\027(so'\013\030z\
! 8|Zz0jAn\031.\022Wn<)О\016\030\016݁\024NjWn\003\
! ܬm\012\036Z^_\021!KX2a}ā\031_d` L B]c^4\033\035߭zÀ %d\
! 2[9X\005_\025pYC?%Wv~]u\"!lV3\026^YxX2\
! '\017\035P\014\012AMC\003ԩj\007\001\024L\016fCy\017d6\037\"\023\034\021_?Ս?\\\010pk\
! YQfR@C<(\035U8gG\035\035Xm\0352&\003ͩ\012\025eg]x)\0335]\0373*D\
! <\005\012Ll귚^KҚfv6\0076j@]Z#e\010HYEA;'\003X,B\003ڠ\005(\020\
! h\011q\013\023\010$qK>95\025\020@MǷy\036:RVN@Zoꦐ\005w\021S\011QZ\024\"v\
! 5)\016\031ZfWV~I2\023f%l\017r`B\014|#[#Aǭ>&fnE\
! &q[`@\003(R|.\036\\\015oV\033\033on\014\023(,o\002Qxl;5UEH\015`^Ȫ\
! W\005\020\022\012IEKJ\037\0018h,\032]\036^^yü͜m,ozz:~\025KT\
! \013\023O[,!|i^\0363\034\033\002pT0]5(N\"f\
! H\030!聢c\004UZtb\004GR4`ZJ{\0012S\015XժQz\020\007\032\032OA\020=cNf\
! Z\026\005b\006\001\027\021?i;A.bh\005K1\023@uZS\013F5E92\015{(rL\003\021/D\
! U(\017x\026!#\027p\016~\0220\007\001\015Uk5\\9\034\034-eO*05,W\035t\
! k\017C%(d\034}\010;F3\0368NT\004ABO\020\011s\027Kf\
! \033*cz6rn eSosiG\014\032y:/ /˨0\034̥K-o'\0114w(\
! /Y,Q\001Ep\000\025\034\0056 b\033\0077\027Rm?CO\035𺵪\035tcF̦eF\
! \000YCRE\031\026F1{޺pxo\027/&/~92Q&'\023\005v{ؒ6`1:W'P\006\
! \022\001򑳧OY \026ɼX0\035M?6HVvjN5\027AEE쌚\016_߬\
! ;U9\004崷\005Do\003\001\025cwg\024\007Z4#:?g8ު\000ŗ(5x߯&O5=;(\
! \033\022\005o{\0377\023$T\005R_W֣\014^\037HYvDw\023oGI,\000\
! ^\025D񶜚ո$\033\016\0258Q[\000X\021\011\031\031o̪T\035Q\035$D\033is+P\
! \017WMN:V#tƪ\012 \032ܚ\034jv\033_Z1\004\002\"nѐ\033.\
! 熚\023u\037p+WkUS[$;5\010ksQ/\003\027\00303ϣF~\026O\
! _@\011|)sr;I\005}1\"\037aω\031\000\011D\026?\003ELUč\\I;s\
! C˶М\002Ȩgc\025bro`\021\011v\025%!$nvea\003>d\022nap\030\
! z&f\027\017cbڄ5\017H\016̧4\002u\021c+-s2\031|\037\016\012\0321UX\004j\
! \0036\032[#;f\037x4tEd;\016xH\032?P/\"cZ?կn \027*0\
! ~\\?\011\033Je\001SZc\014.\003\030f?i+7'=M㨿\
! 0j\014z/\020_\010s\037DwL}\013\031[&]y\001ϯ\012\001K\035(`d,]\
! g!UISOkf7\011m2%\017\031\000*s*$P'\004S4e\031\027Qʳ:\
! \014y*uEq\026n\007\037\003@8}C\011 #7T8Al萂\
! j!*\017\004aV\022%m\025|Bc{\022\027\010\011{%\017@c&.5ޭ3KJ\003\
! )j\001\013dS)ЛMKo6R-z;\013]&g4J\011aTO<\034\017]8w*P\003rFG\
! -\0042)uhN\011ֻ,\023KyOg\004j;':W2\013\024٣j4\006|O|\005T/\
! @\013aC0<{\033QM\001\022ssV~Uc\011\030>1\000'>cN,So\
! <\0110i\006G (0!\021!7ns\021=J&.X1eHa7yj\016\006Kp\034\
! (\037ԉ)u:\000]\015|*b\017LE+\026i\0268TQwz(JS\030ݑX7j^?\
! H31\027\017\033I\030K3\0331/cW\017x\005wh?{ࣿ\024\\\001[\0217ީqbâgj\
! eXM\030&v涢g{\037edu?z\026\0053\\U5ܑ@L$AՉ&qyH\015\017h]`\
! \016m\031\004:o\002Y\025yT慻\035u*͑g3\001SV\016D\016\006V\0276\
! TR\024:mlFv\003\0036/A\006FrsS\017&[X\017PU\023s\016c?^3\
! Q$Vԙ6n\0144\000w8\034ղ55ᬃo;1\026ԗg\000U\007u΁jJc\004\011\012\
! >;QW)Ŷ\002ߎ\016m_*6sB>Rpn@x\025itt@\
! UMjҢ\"t\032Bיi\024G׮#J:\"ʯӵȠ\011!یCT1u|o\037~\\a\0353\
! =x\033Ҳ\015Rq@B\034䪓W#\004\007\"\004\007,2\022'\031Q#O:\034C{2_H\
! iX3G'Ag$in}2\030:U5uC*ufQ599~{7j4m\027=c\033\003\
! }ݪdܡ+i=;cXFH@O;\023!\020ǽzwIT\004)k3]K\
! [\021\012\000O\006K}l\007\000`p\000BT\013\022֕U~u\030r\001(\
! C=\033\031kf#-}5\010/\001\026h@8O׶Wq6\014ᕾ\"a彭\
! 4x\027\0128kh[#k:.5РF\024e\007\002W1jl\016MMUz8*}g|\
! :gl.`EM\006TU=g2\034%d>\034F;vT[ю\007\"t\\\033\
! tb\017\023w9Q\013s\014oKy\000\034\017\003t. \031*Rw7n>z_b\
! !>r?LoEl\027p2y+Nm\013\033\025x`2\025:u\011\
! ?}φ\025M\036=!\015k\027w)ϖw\035-y4s\0158\016M\
! g\022蹩\007[kP7\030\01665\027>SjY1q&=tю]Pj\022\
! \000՞~\023Z\023G:\024*t\007\015M`J\013=}]W\002:j5\037RD_Y\
! IC\034\014͐rd\"RC\"O|W}8\032!A[Pҕ3\031\003ַ8\
! #97 l\024kMշ\0223E\021+&\030yubHkk\011RWɿX\
! #vw*c'%\005FZ^8u:ۿp2\0161*KpoFUċ\021\036~\
! rk\031|jf1;ub\013M~5/4}ָ(7lcS}|Q\026\\%};\016VQ:'ߑ\
! ~2>_ozB\002E\034{YǾ\006b=InFW8VKg]}`\
! xȠ4c\015\003_\014VF{}պ؉g\001^㿅/E=6:\031{'#<:w\031\031t\
! $Jj\0324%[sxRG,.\034\001\030n@\00383s4L\020l \014\
! c(\005\016\010ζt6t+}UwdnSSf\032a\036ŭ\013\025Q}\033\
! F\005x")
  ### end
--- 6,104 ----
  
  import zlib
! data = zlib.decompress("x\\{s8;`8Ю%ʏl.q%Ԕ\013\"!\033aٗ~\015\004\037\
! z1sU$>FCw\003w/^N~}wŮ'o^wx9zq2Fl\
! $H&<\036\016~vo9_\024˘\005\017񻈊X(poSG_rWI!\031\017\012`(\
! 8[\024Ez(YF\027%\017\026aIq$0P\036(\023Uc(\017\030\037`*\
! \003\020(%Q2f\036׷\024|\012ۡ\036\026l\011e\033\004(\000\003\031l̾/||\037\
! pۅ\001=N\014\015(\030\035D4_\000S\031T>\022+jS$\001>y\033LyP#S\036|g\
! LBh)\004tR\036Q28.\017c1\003\032ӳoaTf\016\013«\022ƃMcWn*B.;Eo\
! [bxΣ\026PI۵Gb)υ\032\002\005W<]Ȕ\034\031E6dZh3\017\036;c\
! (,\026\007O1mp6q0bKddL}G^U^7\017$A\017:\030\014htz5#\035\
! WX]V\015\024Ba?,JŝLԀ6+\002\0135u\034yO*?\020~\017\015V\0175v\
! %\011\021\023/\034z?\017{\026͗Q\
! \030\035Oi\001\037@AM7F4l,dX\006|@bj\003́%yqhN\015\010LI\024%$P,\024\001x\005HG&TЯ\015GIZ\026d\016u\021\025\000w\
! \015\001KQv\030څ36z\030\014r\036ei\011V!\030yƗK^(%\0337zVI\007x֝y\
! \024`\012x\007tf>Uu_\027\016!w>7sYaulzhT${\006\0039YD\
! ý\035)2\000B5\003\003\004*+!]ɤ\031\012\022\033ԣk\006\032\027g_\015&E\030\"\017#.Y\\\014\032\
! ѹ\\d.BKvgC$֦6E8QF0W*߂vr]䉫\027\021\006+8|*\
! \005\027z\037\034\007J\003&3\000%5*jL.Y\036d\000<\024[ϓ;\007!\004*\00704\001\
! ?}4\012j\005('l\034\013!\003V`\034\010֎Ro5p2/《d\000Xp\022!\
! \005sQq9\015fG&x\006\027\"\006\"\012;\023k-,(!\017N,\030eHW-+Q4\032G\034\
! \034\005œ׎&kv]\0268\001F=nYE柃\027j[U_?vz\013\010|(j\
! \021\027!X\021Pt#PKhE \020\002)E\037\035PWE @ڳ@e~e\032b7\027eM]sa\032C\
! ixȧu;\015=h9k\015\032$,\025ysƓ\002\034H\017,\011Mv:\
! `Җ\001\012;\025\021\002.,\030GE\006\006\000\007ڧٰn[\006\004)!\
! A7˵vV\015.4kg \032SޯWQV$fͪjF҂\0136+\026\017\013^\
! \003KP\020\000u*ҁ\030\014Zɗf>2A\004=\0353j?[ul춛Ut\
! h\011M\022\0133\0252#}9&X\012r!$\024*WyTw!?\002/\011ѾtK\
! \005=Us\033\"1ۘ*8u,,Nkڪc\027\024DYB\030zPx\012„H\034;Ӳ(@\0263\010N\005шAq\004\011Rv\0244 \
! \0243\014t^H`TUT\003z0\"ǔ\022BDk\\Uz\031\001B6dD\035\001\027\0235\004\031PO8\
! SV=ɕ\003Q\037i3\\\\r@J0TbJ߇i4c[ZKL@\034\
! \017ן&\026z|\0368E\024\0200̓_0\\2#?\\ȹ@\002ꫫN\035k8q\
! \033(X~].p0\026/hܾU)ܽCy2g!h r\017H历3p\027.\001I\0268\
! ž=n\031J~jU\010\033bf.[6\032,JM0\"\032s~||7\
! *\023er2Q`5i\010\026\023\037k~\031\013eP.!\036 \0379{5\014b̋\005/~]cSde\
! \027~?\024c{\021\024XΩn\000\036/\037CCOd@\004k\026>\036Ћ=wbG\
! G\027\027\030[\000~!J\015iSg`/*\016\012ZDv\011~\004\011U\001\024=}Wu\
! /pu.eG4|v5I\012\025K\002T\024Q-ޖS3\033Qiá\002'c&\000+9\
! 5\015P]6ἓШ,5s\005\036溳\011\006T]ǪNs_\025\025\010Y\005Q\001]\037D\\\020\
! EunK+\006@@\032-\032շPspn\036\025oj\023xx={\006\
! q\015u.{\000b\000\006\034__8VgQ\002p\
! &\020)R[I\007\032ʉX\006\033\023f+Im\0359WQ\031*\011wD\
! \"\022:JB\000I\\8ɿ\007}\011)\\\0161\014.n\020Ŵ\011kN\034\037(}DYT5h\004L\
! #Vje2\035c?\034\0254}\010Ԙ\027\007l4b\"eGwF?hԓ\033\000\027\0122\
! M8\031\017#i@=\
! .ހn\"\">7TZ Ik4fAPNeq\0070+\011Ⓐ\014\031|Bc{\023'\010\011[)\017\
! @c&\027.\034%֭OJ\003)j\001\023dS)ЛMKo7R-z;K]&g4\
! J\011aT\017<\0347p8w*q\004rF6\0042uhN\011+,\023KyO\015Mo;':W\
! 2\013\024٣4em\015H\013ĩ\0364;\026†^a8=}\\\0009Fp]+pN#k\
! \002Fk\017z&@\011&7\00355ԋz=O\002fAh/\021H<\012LoHDwȍۜtD\017`.\\\
! ֝4D\005+\014)&R`\0372\003\022:1e\\\003U]̧\"v\027xô^kF1\
! C\033ub4uS\037Mg4\023s񰡛#5i:x#e3v{gpE\
! \012\023>K\001υq7)6LjzZWalgn+z\0147!ZK\037ӬkQ0ݵ\
! \036U\035y\015dD\022Ti\032\005ƽ\026̟A@f.UY\007Le^1_/[\034y\
! f\031z\015Hgǧ[1^kcv8EV./d70޵e7HHsq*d\
! \013\001J}BvNq38gZk&*Ċ\037Vٍ\006ֱ\016#Z&um'F?\021\
! ڂ\022\014N9PM7 TGW;jj\0333eV8kI?\
! )nrF;\005N |\027\0225lI-SZTNN!.S>PuDIu0Q\
! S\027\0314!dq*\015Ïk\0329F\021oCZ\026Dz9il9\007zs+\021\003\021\003L\026L{\
! y}ӓ'd!=S/$r4ԓcz91~Cuk*U*ݥ\
! rrlfP\\_i.v{JDŽ7\0062UڹCWztzͱͣv\002')Cz\013\
! \020ǽrwIT\004O\031K.ץȭ\015Gi\005el'Tͥ\002P\0000EN\000!\
! ^F^BY]w=h<\000u!O5j\\Ö\032I\007\012Ab\0134O ۨkk[\
! +`J7h8\033p6Hkyov7\015ޅv}ڮ8VIeZ\013j\0154h\021E%\
! f̧\032g<\015e~SS2Jߙ4N\0313?\013\013Xvӿ\00158Uw=\031\016\022\0232EOc\
! \016f\035;hy\011kGM;OiRڱDՎB9'ed{g\000Ն]ڗh\
! vxf\025)N;;;]GDob&޶?LoEl!s*2yUdDIɍ]\
! \013YȓL\
! OuEYr8SE\022j~F\036IpXn>-\013\0052K\0279(Ow\003}\015\
! zޓՌޟj͖\"`Ƞ4ch_\014VF7}\024:\010g\001^3\
! 4L@\017\003v;\031\037\037ֹ#Ƞ{FǛWՠyb\004ۓ:b1g\010ƀ?NߑHD*:\
! ݷGsΏ~&\011\026\015d\0164Vwl\035\001\001ٖΦϔ\025\032\0247S\012Cx\
! Lẹ\021}XB\0220\024\031>_6S\016Ŋ\020n")
  ### end



From anadelonbrin at users.sourceforge.net  Tue Sep 30 01:23:55 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Tue Sep 30 01:23:59 2003
Subject: [Spambayes-checkins] spambayes/windows autoconfigure.py,1.5,1.6
Message-ID: 

Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv2300/windows

Modified Files:
	autoconfigure.py 
Log Message:
Add some (limited) smarts to find the expected location of the
various configuration files.  Now to autoconfigure you should
only have to call configure(mailer), where mailer is the name
of one of the mailers the script is aware of.  This coud probably
be wrapped into the install process somehow (maybe a gui
around it?  I don't know what the Inno scripting is like).

Index: autoconfigure.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/autoconfigure.py,v
retrieving revision 1.5
retrieving revision 1.6
diff -C2 -d -r1.5 -r1.6
*** autoconfigure.py	9 Sep 2003 08:55:51 -0000	1.5
--- autoconfigure.py	30 Sep 2003 05:23:53 -0000	1.6
***************
*** 3,10 ****
  """Automatically set up the user's mail client and SpamBayes.
  
  Currently works with:
   o Eudora (POP3/SMTP only)
   o Mozilla Mail (POP3/SMTP only)
!  o Opera Mail (M2) (POP3/SMTP only)
   o Outlook Express (POP3/SMTP only)
   o PocoMail (POP3/SMTP only)
--- 3,14 ----
  """Automatically set up the user's mail client and SpamBayes.
  
+ Example usage:
+     >>> configure("mailer name")
+ Where "mailer name" is any of the names below.
+ 
  Currently works with:
   o Eudora (POP3/SMTP only)
   o Mozilla Mail (POP3/SMTP only)
!  o M2 (Opera Mail) (POP3/SMTP only)
   o Outlook Express (POP3/SMTP only)
   o PocoMail (POP3/SMTP only)
***************
*** 21,26 ****
     only one is necessary.  We should check the existing proxies before
     adding a new one.
-  o Figure out Outlook Express's pop3uidl.dbx file and how to hook into it
-    (use the oe_mailbox.py module)
   o Other mail clients?  Other platforms?
   o This won't work all that well if multiple mail clients are used (they
--- 25,28 ----
***************
*** 31,35 ****
     something does wrong, it's corrupted.  We also write into the file,
     rather than a temporary one and then copy across.  This should all be
!    fixed.
   o Suggestions?
  """
--- 33,42 ----
     something does wrong, it's corrupted.  We also write into the file,
     rather than a temporary one and then copy across.  This should all be
!    fixed.  Richie's suggestion is for the script to create a clone of an
!    existing account with the new settings.  Then people could test the
!    cloned account, and if they're happy with it they can either delete
!    their old account or delete the new one and run the script again in
!    "modify" rather than "clone" mode.  This sounds like a good idea,
!    although a lot of work...
   o Suggestions?
  """
***************
*** 63,67 ****
  import ConfigParser
  
! # Allow for those without SpamBayes on the PYTHONPATH
  sys.path.insert(-1, os.getcwd())
  sys.path.insert(-1, os.path.dirname(os.getcwd()))
--- 70,74 ----
  import ConfigParser
  
! # Allow for those without SpamBayes on their PYTHONPATH
  sys.path.insert(-1, os.getcwd())
  sys.path.insert(-1, os.path.dirname(os.getcwd()))
***************
*** 114,118 ****
              if c.get(sect, "UsesIMAP") == "0":
                  # Eudora stores the POP3 server name in two places.
!                 # Why?  Who knows?  We do the popaccount one
                  # separately, because it also has the username.
                  p = c.get(sect, "popaccount")
--- 121,125 ----
              if c.get(sect, "UsesIMAP") == "0":
                  # Eudora stores the POP3 server name in two places.
!                 # Why?  Who cares.  We do the popaccount one
                  # separately, because it also has the username.
                  p = c.get(sect, "popaccount")
***************
*** 413,417 ****
      # here.
  
! def configure_outlook_express():
      """Configure OE to use the SpamBayes POP3 and SMTP proxies, and
      configure SpamBayes to proxy the servers that OE was connecting to."""
--- 420,424 ----
      # here.
  
! def configure_outlook_express(unused):
      """Configure OE to use the SpamBayes POP3 and SMTP proxies, and
      configure SpamBayes to proxy the servers that OE was connecting to."""
***************
*** 546,550 ****
      rules_file.close()
  
! def configure_pocomail():
       import win32api
       import win32con
--- 553,557 ----
      rules_file.close()
  
! def configure_pocomail(unused):
       import win32api
       import win32con
***************
*** 657,660 ****
--- 664,714 ----
  
  
+ def find_config_location(mailer):
+     """Attempt to find the location of the config file for
+     the given mailer, to pass to the configure_* scripts
+     above."""
+     import win32api
+     from win32com.shell import shell, shellcon
+     if mailer in ["Outlook Express", "PocoMail"]:
+         # Outlook Express and PocoMail can be configured without a
+         # config location, because it's all in the registry
+         return ""
+     windowsUserDirectory = shell.SHGetFolderPath(0,shellcon.CSIDL_APPDATA,0,0)
+     potential_locations = {"Eudora" : ("Qualcomm%(sep)sEudora",),
+                            "Mozilla" : ("Mozilla%(sep)sProfiles%(sep)s%(user)s",
+                                         "Mozilla%(sep)sProfiles%(sep)sdefault",),
+                            "M2" : ("Opera%(sep)sOpera7",),
+                            }
+     # We try with the username that the user uses [that's a lot of 'use'!]
+     # for Windows, even though that might not be the same as their profile
+     # names for mailers.  We can get smarter later.
+     username = win32api.GetUserName()
+     loc_dict = {"sep" : os.sep,
+                 "user" : username}
+     for loc in potential_locations[mailer]:
+         loc = loc % loc_dict
+         loc = os.path.join(windowsUserDirectory, loc)
+         if os.path.exists(loc):
+             return loc
+     return None
+ 
+ 
+ def configure(mailer):
+     """Automatically configure the specified mailer and SpamBayes.
+     Return True if successful, False otherwise.
+     """
+     loc = find_config_location(mailer)
+     if loc is None:
+         return False
+     funcs = {"Eudora" : configure_eudora,
+              "Mozilla" : configure_mozilla,
+              "M2" : configure_m2,
+              "Outlook Express" : configure_outlook_express,
+              "PocoMail" : configure_pocomail,
+              }
+     funcs[mailer](loc)
+     return True
+ 
+ 
  if __name__ == "__main__":
      pmail_ini_dir = "C:\\Program Files\\PMAIL\\MAIL\\ADMIN"
***************
*** 665,667 ****
      #configure_pocomail()
      #configure_pegasus_mail(pmail_ini_dir)
!     pass
--- 719,722 ----
      #configure_pocomail()
      #configure_pegasus_mail(pmail_ini_dir)
!     for mailer in ["Eudora", "Mozilla", "M2", "Outlook Express", "PocoMail"]:
!         print find_config_location(mailer)



From tim_one at users.sourceforge.net  Tue Sep 30 10:37:46 2003
From: tim_one at users.sourceforge.net (Tim Peters)
Date: Tue Sep 30 10:37:49 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000 oastats.py,1.1,1.2
Message-ID: 

Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv5221/Outlook2000

Modified Files:
	oastats.py 
Log Message:
GetStats():  Round percentages in display.  %d truncates, and it's
disconcerting to see the percentages add up to 98% (as I just did ).


Index: oastats.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/oastats.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** oastats.py	29 Sep 2003 02:14:25 -0000	1.1
--- oastats.py	30 Sep 2003 14:37:44 -0000	1.2
***************
*** 44,50 ****
          format_dict.update(self.__dict__)
          push("SpamBayes has processed %(num_seen)d messages - " \
!              "%(num_ham)d (%(perc_ham)d%%) good, " \
!              "%(num_spam)d (%(perc_spam)d%%) spam " \
!              "and %(num_unsure)d (%(perc_unsure)d%%) unsure" % format_dict)
          if self.num_recovered_good:
              push("%(num_recovered_good)d message(s) were manually " \
--- 44,50 ----
          format_dict.update(self.__dict__)
          push("SpamBayes has processed %(num_seen)d messages - " \
!              "%(num_ham)d (%(perc_ham).0f%%) good, " \
!              "%(num_spam)d (%(perc_spam).0f%%) spam " \
!              "and %(num_unsure)d (%(perc_unsure).0f%%) unsure" % format_dict)
          if self.num_recovered_good:
              push("%(num_recovered_good)d message(s) were manually " \